From d2f8cade709b7d55a93e18592b6e38247d648ca9 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Mon, 27 Mar 2017 19:56:00 -0400 Subject: limit cbt memory usage. circle is limited to 4GB --- cbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cbt b/cbt index 06112d1..53fe9f4 100755 --- a/cbt +++ b/cbt @@ -228,7 +228,7 @@ stage1 () { log "Running JVM directly" "$@" options=($JAVA_OPTS) # JVM options to improve startup time. See https://github.com/cvogt/cbt/pull/262 - java "${options[@]}" $DEBUG -Xmx6072m -Xss10M -XX:MaxJavaStackTraceDepth=-1 -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -Xverify:none -cp "$NAILGUN$TARGET" cbt.NailgunLauncher "$(time_taken)" "$CWD" "$loop" "$@" + java "${options[@]}" $DEBUG -Xmx3072m -Xss10M -XX:MaxJavaStackTraceDepth=-1 -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -Xverify:none -cp "$NAILGUN$TARGET" cbt.NailgunLauncher "$(time_taken)" "$CWD" "$loop" "$@" exitCode=$? else log "Running via background process (nailgun)" "$@" -- cgit v1.2.3 From bba2abe7ee38b8903822a07578c46466923d13ed Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Mon, 20 Mar 2017 22:09:38 -0400 Subject: start modularizing cbt into libraries this extracts certain parts of cbt into stand-alone libraries, which can be published to maven and used outside of cbt. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This also adds scalariform for these parts of the code. This slows down cbt’s own build a lot because of the number of projects involved! So we’ll follow this by a bunch of performance tweak commits. --- build/build.scala | 9 +- build/build/build.scala | 2 +- cbt | 2 +- doc/design.md | 35 +++++ internal/plugins/library/Library.scala | 51 +++++++ internal/plugins/library/build/build.scala | 7 + internal/plugins/shared/Shared.scala | 2 +- libraries/capture_args/build/build.scala | 12 +- libraries/capture_args/build/build/build.scala | 5 + libraries/capture_args/package.scala | 28 ++-- libraries/common-0/ProxySecurityManager.java | 130 ++++++++++++++++++ libraries/common-0/TrapSecurityManager.java | 88 ++++++++++++ libraries/common-0/TrapSystemExit.java | 35 +++++ libraries/common-0/build/build.scala | 7 + libraries/common-0/build/build/build.scala | 5 + libraries/common-1/ExitCode.java | 24 ++++ libraries/common-1/ExitCode.scala | 11 ++ libraries/common-1/build/build.scala | 8 ++ libraries/common-1/build/build/build.scala | 5 + libraries/common-1/common_1.scala | 34 +++++ libraries/file/build/build.scala | 8 ++ libraries/file/build/build/build.scala | 5 + libraries/file/file.scala | 112 ++++++++++++++++ libraries/interfaces/ExitCode.java | 4 + libraries/proguard/Proguard.scala | 10 +- libraries/proguard/build/build.scala | 8 +- libraries/proguard/build/build/build.scala | 8 +- libraries/reflect/StaticMethod.scala | 4 + libraries/reflect/build/build.scala | 8 ++ libraries/reflect/build/build/build.scala | 5 + libraries/reflect/reflect.scala | 177 +++++++++++++++++++++++++ nailgun_launcher/NailgunLauncher.java | 29 ++-- nailgun_launcher/ProxySecurityManager.java | 102 -------------- nailgun_launcher/Stage0Lib.java | 33 ++--- nailgun_launcher/TrapSecurityManager.java | 83 ------------ plugins/proguard/Proguard.scala | 2 +- plugins/uber-jar/src/UberJar.scala | 2 +- stage1/Stage1Lib.scala | 125 ++++------------- stage1/cbt.scala | 62 ++------- stage1/resolver.scala | 29 ++-- stage2/BasicBuild.scala | 11 +- stage2/BuildBuild.scala | 18 +-- stage2/DirectoryDependency.scala | 2 +- stage2/Lib.scala | 58 +------- stage2/libraries.scala | 12 ++ stage2/plugins.scala | 17 +++ stage2/plugins/Dotty.scala | 6 +- test/build/build.scala | 1 + 48 files changed, 945 insertions(+), 496 deletions(-) create mode 100644 internal/plugins/library/Library.scala create mode 100644 internal/plugins/library/build/build.scala create mode 100644 libraries/capture_args/build/build/build.scala create mode 100644 libraries/common-0/ProxySecurityManager.java create mode 100644 libraries/common-0/TrapSecurityManager.java create mode 100644 libraries/common-0/TrapSystemExit.java create mode 100644 libraries/common-0/build/build.scala create mode 100644 libraries/common-0/build/build/build.scala create mode 100644 libraries/common-1/ExitCode.java create mode 100644 libraries/common-1/ExitCode.scala create mode 100644 libraries/common-1/build/build.scala create mode 100644 libraries/common-1/build/build/build.scala create mode 100644 libraries/common-1/common_1.scala create mode 100644 libraries/file/build/build.scala create mode 100644 libraries/file/build/build/build.scala create mode 100644 libraries/file/file.scala create mode 100644 libraries/interfaces/ExitCode.java create mode 100644 libraries/reflect/StaticMethod.scala create mode 100644 libraries/reflect/build/build.scala create mode 100644 libraries/reflect/build/build/build.scala create mode 100644 libraries/reflect/reflect.scala delete mode 100644 nailgun_launcher/ProxySecurityManager.java delete mode 100644 nailgun_launcher/TrapSecurityManager.java create mode 100644 stage2/libraries.scala create mode 100644 stage2/plugins.scala diff --git a/build/build.scala b/build/build.scala index c5ad1b7..e077343 100644 --- a/build/build.scala +++ b/build/build.scala @@ -1,3 +1,4 @@ +package cbt_build.cbt import cbt._ import cbt_internal._ @@ -12,8 +13,8 @@ class Build(val context: Context) extends Shared with Scalariform with PublishLo super.dependencies ++ Resolver(mavenCentral).bind( MavenDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r"), ScalaDependency("org.scala-lang.modules","scala-xml",constants.scalaXmlVersion) - ) - } :+ libraries.eval + ) :+ libraries.reflect :+ libraries.eval + } override def sources = Seq( "nailgun_launcher", "stage1", "stage2", "compatibility" @@ -22,7 +23,9 @@ class Build(val context: Context) extends Shared with Scalariform with PublishLo override def scalariform = super.scalariform.copy( Seq( context.cbtHome / "stage2" / "DirectoryDependency.scala", - context.cbtHome / "stage2" / "LazyDependency.scala" + context.cbtHome / "stage2" / "LazyDependency.scala", + context.cbtHome / "stage2" / "libraries.scala", + context.cbtHome / "stage2" / "plugins.scala" ) ) diff --git a/build/build/build.scala b/build/build/build.scala index 6752f27..44b8b71 100644 --- a/build/build/build.scala +++ b/build/build/build.scala @@ -1,6 +1,6 @@ package cbt_build.cbt.build import cbt._ -class Build(val context: Context) extends CbtInternal{ +class Build(val context: Context) extends BuildBuild with CbtInternal{ override def dependencies = ( super.dependencies :+ cbtInternal.shared :+ plugins.scalariform ) diff --git a/cbt b/cbt index 53fe9f4..61fc58a 100755 --- a/cbt +++ b/cbt @@ -196,7 +196,7 @@ stage1 () { log "Checking for changes in cbt/nailgun_launcher" "$@" NAILGUN_INDICATOR=$NAILGUN$TARGET../classes.last-success changed=1 - NAILGUN_SOURCES=("$NAILGUN"*.java) + NAILGUN_SOURCES=("$NAILGUN"*.java "$CBT_HOME"/libraries/common-0/*.java) for file in "${NAILGUN_SOURCES[@]}"; do if [ "$file" -nt "$NAILGUN_INDICATOR" ]; then changed=0; fi done diff --git a/doc/design.md b/doc/design.md index 28bbdaa..38e5434 100644 --- a/doc/design.md +++ b/doc/design.md @@ -83,6 +83,40 @@ class Build(val context: Context) extends SomePlugin{ Such a simple replacement of `b` while keeping all other arguments would not be easily possible if doSomething was a def not a case class. +## Why do the libraries have ops packages and Module traits? + +Java's and Scala's package system does allow importing things, +but not exporting things. Everything has to be imported +explicitly anywhere it is supposed to be used. It's just a +package system, not a module system. This leads to a lot of +import boiler plate. CBT tries to minimize the imports +necessary for it's use however. So how to we do this while +at the same time allowing modularization? In particular, how +do we do this with stand-alone methods and implicit classes +that have to be in an object, e.g. the package object? + +Scala's traits can be used as a module system that supports exports. +This means we can take several modules (traits) and merge them into +something that exports everything defined in any of them. Basically +inheriting a trait means importing names into the scope of the +inheriting class and exporting those names to the class's users. + +CBT's libraries define Module traits, which their package objects inherit. +This makes it easy to use the libraries by itself. CBT's core however +also inherits all of the library Module traits in it's package object, +meaning that by a simple `import cbt._` you get everything from all +libraries. This solves the import boiler plate. + +For implicit classes it is a little bit trickier as those should +extend AnyVal for performance reasons, but that's only allowed in +an object, not a trait. So what we do instead is put Universal traits +(see http://docs.scala-lang.org/overviews/core/value-classes.html) +containing all the logic into a helper package `ops`. The package +objects that want to offer the implicit classes now define them +extending the Universal traits. This means a little boiler plate +where the package object is define, but also solves the import +boiler plate everywhere else. + ## What is newBuild and why do we need it? Methods in a class can call each other and thereby effectively form a graph. @@ -137,3 +171,4 @@ trait CrossVersionPlugin{ Problem solved. In fact this allows for a very, very flexible way of creating differents variants of your build. + diff --git a/internal/plugins/library/Library.scala b/internal/plugins/library/Library.scala new file mode 100644 index 0000000..a9dec7c --- /dev/null +++ b/internal/plugins/library/Library.scala @@ -0,0 +1,51 @@ +package cbt_internal +import cbt._ +import java.io._ +import scala.concurrent._ +import scala.concurrent.duration._ +trait Library extends Scalariform with GoogleJavaFormat with DynamicOverrides with AdvancedScala{ + def inceptionYear: Int + def description: String + def version = ??? + override def compile = { + googleJavaFormat() + scalariform() + super.compile + } + + def publishIfChanged = newBuild[PublishIfChanged]({s""" + def inceptionYear = $inceptionYear + def description = ${description.quote} + def apply = if(changedInMaster) publish + """}) +} + +trait PublishIfChanged extends PackageJars with DynamicOverrides with Shared{ + override def url = super.url ++ "/libraries/" ++ name + + def gitHash = { + val p = new ProcessBuilder( + "git rev-parse HEAD".split(" "): _* + ) + .directory( projectDirectory ) + .start + + val sout = new InputStreamReader(p.getInputStream); + import scala.concurrent.ExecutionContext.Implicits.global + val out = Future(blocking(Iterator.continually(sout.read).takeWhile(_ != -1).map(_.toChar).mkString)) + p.waitFor + val revision = Await.result( out, Duration.Inf ).trim + revision + } + override def version = "rev-"++gitHash + + def changedInMaster = ( + 0 === + new ProcessBuilder( + "git diff --exit-code --quiet master..master^ .".split(" "): _* + ) + .directory( projectDirectory ) + .start + .waitFor + ) +} diff --git a/internal/plugins/library/build/build.scala b/internal/plugins/library/build/build.scala new file mode 100644 index 0000000..6b9432e --- /dev/null +++ b/internal/plugins/library/build/build.scala @@ -0,0 +1,7 @@ +package cbt_build.cbt_internal.library_build_plugin +import cbt._ +class Build(val context: Context) extends Plugin with CbtInternal{ + override def dependencies = ( + super.dependencies :+ cbtInternal.shared :+ plugins.scalariform :+ plugins.googleJavaFormat + ) +} diff --git a/internal/plugins/shared/Shared.scala b/internal/plugins/shared/Shared.scala index 90bc4b2..2db9770 100644 --- a/internal/plugins/shared/Shared.scala +++ b/internal/plugins/shared/Shared.scala @@ -1,7 +1,7 @@ package cbt_internal import cbt._ import java.net.URL -trait Shared extends SonatypeRelease with SnapshotVersion with GithubPom{ +trait Shared extends AdvancedScala with SonatypeRelease with SnapshotVersion with GithubPom{ override def user = "cvogt" override def groupId = "org.cvogt" override def organization = Some( Organization( "Jan Christopher Vogt", Some( new URL("http://cvogt.org") ) ) ) diff --git a/libraries/capture_args/build/build.scala b/libraries/capture_args/build/build.scala index 24c1faa..fab36bf 100644 --- a/libraries/capture_args/build/build.scala +++ b/libraries/capture_args/build/build.scala @@ -1,11 +1,21 @@ package cbt_build.cbt.capture_args import cbt._ -class Build(val context: Context) extends BaseBuild{ +import cbt_internal._ +class Build(val context: Context) extends Library{ + def description = ( + "macro that allows you to extract a functions arguments" + ++" as strings in order to programmatically pass them to a stringly typed" + ++" api such as a process call, http or a .main method" + ) + + def inceptionYear = 2017 + override def dependencies = ( super.dependencies ++ // don't forget super.dependencies here for scala-library, etc. Resolver( mavenCentral ).bind( MavenDependency( "org.scala-lang", "scala-reflect", scalaVersion ) ) ) + override def scalacOptions = super.scalacOptions :+ "-language:experimental.macros" } diff --git a/libraries/capture_args/build/build/build.scala b/libraries/capture_args/build/build/build.scala new file mode 100644 index 0000000..6fabf47 --- /dev/null +++ b/libraries/capture_args/build/build/build.scala @@ -0,0 +1,5 @@ +package cbt_build.cbt.capture_args.build +import cbt._ +class Build(val context: Context) extends BuildBuild with CbtInternal{ + override def dependencies = super.dependencies :+ cbtInternal.library +} diff --git a/libraries/capture_args/package.scala b/libraries/capture_args/package.scala index f5cd219..5c96a8d 100644 --- a/libraries/capture_args/package.scala +++ b/libraries/capture_args/package.scala @@ -2,29 +2,29 @@ package cbt.capture_args import scala.reflect._ import scala.reflect.macros.blackbox.Context -case class Argument( annotations: Seq[annotation.Annotation], name: String, values: Option[Seq[String]] ){ +case class Argument( annotations: Seq[annotation.Annotation], name: String, values: Option[Seq[String]] ) { def toSeqOption = values.map( name +: _ ) } case class Signature( name: String, args: Seq[Argument] ) -object `package`{ - def captureArgsImplementation(c: Context): c.Tree = { +object `package` { + def captureArgsImplementation( c: Context ): c.Tree = { import c.universe._ - def literal( a: Any ) = Literal(Constant(a)) - def ident( name: String ) = Ident(TermName(name)) + def literal( a: Any ) = Literal( Constant( a ) ) + def ident( name: String ) = Ident( TermName( name ) ) - def findOwnerRecursive(symbol: Symbol, predicate: Symbol => Boolean): Option[Symbol] = { - Option(symbol).flatMap{ + def findOwnerRecursive( symbol: Symbol, predicate: Symbol => Boolean ): Option[Symbol] = { + Option( symbol ).flatMap { s => - if(s == NoSymbol) None else if(predicate(s)) Some(s) else findOwnerRecursive(s.owner, predicate) + if ( s == NoSymbol ) None else if ( predicate( s ) ) Some( s ) else findOwnerRecursive( s.owner, predicate ) } } val method: MethodSymbol = ( - findOwnerRecursive(c.internal.enclosingOwner, _.isMethod).map(_.asMethod) + findOwnerRecursive( c.internal.enclosingOwner, _.isMethod ).map( _.asMethod ) orElse - findOwnerRecursive(c.internal.enclosingOwner, _.isClass).map(_.asClass.primaryConstructor.asMethod) + findOwnerRecursive( c.internal.enclosingOwner, _.isClass ).map( _.asClass.primaryConstructor.asMethod ) getOrElse { c.error( c.enclosingPosition, @@ -33,14 +33,14 @@ object `package`{ ??? } ) - val name = literal(method.name.decodedName.toString) + val name = literal( method.name.decodedName.toString ) // Note: method.paramLists requires explicitly annotated result type - val params = method.paramLists.flatten.map(_.asTerm) + val params = method.paramLists.flatten.map( _.asTerm ) - val args = params.map{ s => + val args = params.map { s => val name = literal( s.name.decodedName.toString ) val i = ident( s.name.toString ) - q"_root_.cbt.capture_args.Argument( _root_.scala.Seq( ..${s.annotations.map(_.tree)} ), $name, valueToStrings($i) )" + q"_root_.cbt.capture_args.Argument( _root_.scala.Seq( ..${s.annotations.map( _.tree )} ), $name, valueToStrings($i) )" } val tree = q""" _root_.cbt.capture_args.Signature( name = $name, args = Seq( ..$args ) ) diff --git a/libraries/common-0/ProxySecurityManager.java b/libraries/common-0/ProxySecurityManager.java new file mode 100644 index 0000000..4669add --- /dev/null +++ b/libraries/common-0/ProxySecurityManager.java @@ -0,0 +1,130 @@ +package cbt.reflect; + +import java.security.*; +import java.io.FileDescriptor; +import java.net.InetAddress; + +/* +SecurityManager proxy that forwards all calls to the provided target if != null. +Useful to replace a previously installed SecurityManager, overriding some methods +but forwarding the rest. +*/ +class ProxySecurityManager extends SecurityManager { + private SecurityManager target; + + protected ProxySecurityManager(SecurityManager target) { + this.target = target; + } + + public Object getSecurityContext() { + if (target != null) return target.getSecurityContext(); + else return super.getSecurityContext(); + } + + public void checkPermission(Permission perm) { + if (target != null) target.checkPermission(perm); + } + + public void checkPermission(Permission perm, Object context) { + if (target != null) target.checkPermission(perm, context); + } + + public void checkCreateClassLoader() { + if (target != null) target.checkCreateClassLoader(); + } + + public void checkAccess(Thread t) { + if (target != null) target.checkAccess(t); + } + + public void checkAccess(ThreadGroup g) { + if (target != null) target.checkAccess(g); + } + + public void checkExit(int status) { + if (target != null) target.checkExit(status); + } + + public void checkExec(String cmd) { + if (target != null) target.checkExec(cmd); + } + + public void checkLink(String lib) { + if (target != null) target.checkLink(lib); + } + + /* + public void checkRead(FileDescriptor fd) { + if (target != null) target.checkRead(fd); + } + + public void checkRead(String file) { + if (target != null) target.checkRead(file); + } + + public void checkRead(String file, Object context) { + if (target != null) target.checkRead(file, context); + } + */ + + public void checkWrite(FileDescriptor fd) { + if (target != null) target.checkWrite(fd); + } + + public void checkWrite(String file) { + if (target != null) target.checkWrite(file); + } + + public void checkDelete(String file) { + if (target != null) target.checkDelete(file); + } + + public void checkConnect(String host, int port) { + if (target != null) target.checkConnect(host, port); + } + + public void checkConnect(String host, int port, Object context) { + if (target != null) target.checkConnect(host, port, context); + } + + public void checkListen(int port) { + if (target != null) target.checkListen(port); + } + + public void checkAccept(String host, int port) { + if (target != null) target.checkAccept(host, port); + } + + public void checkMulticast(InetAddress maddr) { + if (target != null) target.checkMulticast(maddr); + } + + public void checkPropertiesAccess() { + if (target != null) target.checkPropertiesAccess(); + } + + public void checkPropertyAccess(String key) { + if (target != null) target.checkPropertyAccess(key); + } + + public void checkPrintJobAccess() { + if (target != null) target.checkPrintJobAccess(); + } + + public void checkPackageAccess(String pkg) { + if (target != null) target.checkPackageAccess(pkg); + } + + public void checkPackageDefinition(String pkg) { + if (target != null) target.checkPackageDefinition(pkg); + } + + public void checkSetFactory() { + if (target != null) target.checkSetFactory(); + } + + public ThreadGroup getThreadGroup() { + if (target != null) return target.getThreadGroup(); + else return super.getThreadGroup(); + } +} diff --git a/libraries/common-0/TrapSecurityManager.java b/libraries/common-0/TrapSecurityManager.java new file mode 100644 index 0000000..161b74f --- /dev/null +++ b/libraries/common-0/TrapSecurityManager.java @@ -0,0 +1,88 @@ +package cbt.reflect; + +import java.security.*; +/* +When enabled, this SecurityManager turns System.exit(...) calls into exceptions that can be caught and handled. +Installing a SecurityManager is a global side-effect and thus needs extra care in a persistent +background process like CBT's. The current approach is install it once during JVM-startup. +When disabled this delegates to the SecurityManager installed before if any, which +would be Nailgun's if running on Nailgun. If we do not delegate to Nailgun, it seems we +could in some cases kill the server process +*/ +public class TrapSecurityManager extends ProxySecurityManager { + public static ThreadLocal trapExitCode() { + // storing the flag in the installed security manager + // instead of e.g. a static member is necessary because + // we run multiple versions of CBT with multiple TrapSecurityManager classes + // but we need to affect the installed one + SecurityManager sm = System.getSecurityManager(); + if (sm instanceof TrapSecurityManager) { + return ((TrapSecurityManager) sm)._trapExitCode; + } else { + try { + @SuppressWarnings("unchecked") + ThreadLocal res = + (ThreadLocal) sm.getClass().getMethod("trapExitCode").invoke(null); + return res; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + + private final ThreadLocal _trapExitCode = + new ThreadLocal() { + @Override + protected Boolean initialValue() { + return false; + } + }; + + protected TrapSecurityManager(SecurityManager parent) { + super(parent); + } + + public void checkPermission(Permission permission) { + /* + NOTE: is it actually ok, to just make these empty? + Calling .super leads to ClassNotFound exteption for a lambda. + Calling to the previous SecurityManager leads to a stack overflow + */ + if (!TrapSecurityManager.trapExitCode().get()) { + super.checkPermission(permission); + } + } + + public void checkPermission(Permission permission, Object context) { + /* Does this methods need to be overidden? */ + if (!TrapSecurityManager.trapExitCode().get()) { + super.checkPermission(permission, context); + } + } + + // FIXME: we should probably choose a more unique name for this + private static final String prefix = "[TrappedExit] "; + + @Override + public void checkExit(int status) { + if (TrapSecurityManager.trapExitCode().get()) { + // using a RuntimeException and a prefix here instead of a custom + // exception type because this is thrown by the installed TrapSecurityManager + // but other versions of cbt need to be able to catch it, that do not have access + // to that version of the TrapSecurityManager class + throw new RuntimeException(prefix + status); + } + super.checkExit(status); + } + + public static boolean isTrappedExit(Throwable t) { + return t instanceof RuntimeException + && t.getMessage() != null + && t.getMessage().startsWith(prefix); + } + + public static int exitCode(Throwable t) { + assert (isTrappedExit(t)); + return Integer.parseInt(t.getMessage().substring(prefix.length())); + } +} diff --git a/libraries/common-0/TrapSystemExit.java b/libraries/common-0/TrapSystemExit.java new file mode 100644 index 0000000..86bc880 --- /dev/null +++ b/libraries/common-0/TrapSystemExit.java @@ -0,0 +1,35 @@ +package cbt.reflect; + +import java.security.*; +import java.lang.reflect.InvocationTargetException; + +public abstract class TrapSystemExit { + public static SecurityManager createSecurityManager(SecurityManager delegateTo) { + return new TrapSecurityManager(delegateTo); + } + + public static T run(TrapSystemExit runnable) throws Throwable { + boolean trapExitCodeBefore = TrapSecurityManager.trapExitCode().get(); + try { + TrapSecurityManager.trapExitCode().set(true); + return runnable.run(); + } catch (InvocationTargetException exception) { + Throwable cause = exception.getCause(); + if (TrapSecurityManager.isTrappedExit(cause)) { + return runnable.wrap(TrapSecurityManager.exitCode(cause)); + } + throw exception; + } catch (Exception exception) { + if (TrapSecurityManager.isTrappedExit(exception)) { + return runnable.wrap(TrapSecurityManager.exitCode(exception)); + } + throw exception; + } finally { + TrapSecurityManager.trapExitCode().set(trapExitCodeBefore); + } + } + + public abstract T run() throws Throwable; + + public abstract T wrap(int exitCode); +} diff --git a/libraries/common-0/build/build.scala b/libraries/common-0/build/build.scala new file mode 100644 index 0000000..0a4e5cb --- /dev/null +++ b/libraries/common-0/build/build.scala @@ -0,0 +1,7 @@ +package cbt_build.common_0 +import cbt._ +import cbt_internal._ +class Build(val context: Context) extends Library{ + override def inceptionYear = 2017 + override def description = "classes shared by multiple cbt libraries and needed in stage 0" +} diff --git a/libraries/common-0/build/build/build.scala b/libraries/common-0/build/build/build.scala new file mode 100644 index 0000000..d3f98ce --- /dev/null +++ b/libraries/common-0/build/build/build.scala @@ -0,0 +1,5 @@ +package cbt_build.reflect.build +import cbt._ +class Build(val context: Context) extends BuildBuild with CbtInternal{ + override def dependencies = super.dependencies :+ cbtInternal.library +} diff --git a/libraries/common-1/ExitCode.java b/libraries/common-1/ExitCode.java new file mode 100644 index 0000000..1c16f67 --- /dev/null +++ b/libraries/common-1/ExitCode.java @@ -0,0 +1,24 @@ +package cbt; +/* +public class ExitCode{ + public int integer; + public ExitCode(int integer){ + this.integer = integer; + } + public static ExitCode apply(int integer){ + return new ExitCode( integer ); + } + public static ExitCode Success = new ExitCode(0); + public static ExitCode Failure = new ExitCode(1); + + @Override + public boolean equals(Object other){ + return (other instanceof ExitCode) && ((ExitCode) other).integer == integer; + } + @Override + public int hashCode(){ + return integer; + } +} + +*/ diff --git a/libraries/common-1/ExitCode.scala b/libraries/common-1/ExitCode.scala new file mode 100644 index 0000000..41d9f3f --- /dev/null +++ b/libraries/common-1/ExitCode.scala @@ -0,0 +1,11 @@ +package cbt +// CLI interop +case class ExitCode( integer: Int ) extends interfaces.ExitCode { + def ||( other: => ExitCode ) = if ( this == ExitCode.Success ) this else other + def &&( other: => ExitCode ) = if ( this != ExitCode.Success ) this else other +} +object ExitCode { + val Success = ExitCode( 0 ) + val Failure = ExitCode( 1 ) +} + diff --git a/libraries/common-1/build/build.scala b/libraries/common-1/build/build.scala new file mode 100644 index 0000000..247fd01 --- /dev/null +++ b/libraries/common-1/build/build.scala @@ -0,0 +1,8 @@ +package cbt_build.common_1 +import cbt._ +import cbt_internal._ +class Build(val context: Context) extends Library{ + override def inceptionYear = 2017 + override def description = "classes shared by multiple cbt libraries and needed in stage 1" + override def dependencies = super.dependencies :+ libraries.common_0 :+ libraries.interfaces +} diff --git a/libraries/common-1/build/build/build.scala b/libraries/common-1/build/build/build.scala new file mode 100644 index 0000000..d3f98ce --- /dev/null +++ b/libraries/common-1/build/build/build.scala @@ -0,0 +1,5 @@ +package cbt_build.reflect.build +import cbt._ +class Build(val context: Context) extends BuildBuild with CbtInternal{ + override def dependencies = super.dependencies :+ cbtInternal.library +} diff --git a/libraries/common-1/common_1.scala b/libraries/common-1/common_1.scala new file mode 100644 index 0000000..66da224 --- /dev/null +++ b/libraries/common-1/common_1.scala @@ -0,0 +1,34 @@ +package cbt.common_1 +import cbt.ExitCode +object `package` extends Module { + implicit class CbtExitCodeOps( val exitCode: ExitCode ) extends AnyVal with ops.CbtExitCodeOps + implicit class TypeInferenceSafeEquals[T]( val value: T ) extends AnyVal with ops.TypeInferenceSafeEquals[T] + implicit class CbtBooleanOps( val condition: Boolean ) extends AnyVal with ops.CbtBooleanOps + implicit class CbtStringOps( val string: String ) extends AnyVal with ops.CbtStringOps +} + +package ops { + trait CbtExitCodeOps extends Any { + def exitCode: ExitCode + def ||( other: => ExitCode ) = if ( exitCode == ExitCode.Success ) exitCode else other + def &&( other: => ExitCode ) = if ( exitCode != ExitCode.Success ) exitCode else other + } + trait TypeInferenceSafeEquals[T] extends Any { + def value: T + /** if you don't manually upcast, this will catch comparing different types */ + def ===( other: T ) = value == other + def =!=( other: T ) = value != other // =!= instead of !==, because it has better precedence + } + trait CbtBooleanOps extends Any { + def condition: Boolean + def option[T]( value: => T ): Option[T] = if ( condition ) Some( value ) else None + } + trait CbtStringOps extends Any { + def string: String + def escape = string.replace( "\\", "\\\\" ).replace( "\"", "\\\"" ) + def quote = s""""$escape"""" + def ~( right: String ): String = string + right + } +} + +trait Module diff --git a/libraries/file/build/build.scala b/libraries/file/build/build.scala new file mode 100644 index 0000000..d9017a1 --- /dev/null +++ b/libraries/file/build/build.scala @@ -0,0 +1,8 @@ +package cbt_build.reflect +import cbt._ +import cbt_internal._ +class Build(val context: Context) extends Library{ + override def inceptionYear = 2017 + override def description = "helpers to work with java io and nio" + override def dependencies = super.dependencies :+ libraries.common_1 +} diff --git a/libraries/file/build/build/build.scala b/libraries/file/build/build/build.scala new file mode 100644 index 0000000..d3f98ce --- /dev/null +++ b/libraries/file/build/build/build.scala @@ -0,0 +1,5 @@ +package cbt_build.reflect.build +import cbt._ +class Build(val context: Context) extends BuildBuild with CbtInternal{ + override def dependencies = super.dependencies :+ cbtInternal.library +} diff --git a/libraries/file/file.scala b/libraries/file/file.scala new file mode 100644 index 0000000..f20c9a8 --- /dev/null +++ b/libraries/file/file.scala @@ -0,0 +1,112 @@ +package cbt.file +import java.io.File +import java.nio.file.Files._ +import java.nio.file.StandardCopyOption._ +import cbt.common_1._ +object `package` extends Module { + implicit class CbtFileOps( val file: File ) extends ops.CbtFileOps +} + +package ops { + trait CbtFileOps extends Any { + def file: File + def ++( s: String ): File = { + if ( s endsWith "/" ) throw new Exception( + """Trying to append a String that ends in "/" to a File would loose the trailing "/". Use .stripSuffix("/") if you need to.""" + ) + new File( string + s ) // PERFORMANCE HOTSPOT + } + def /( s: String ): File = { + new File( file, s ) + } + def parent = realpath( file / ".." ) + def string = file.toString + + def listOrFail: Seq[File] = Option( file.listFiles ).getOrElse( throw new Exception( "no such file: " + file ) ).toVector + def listRecursive: Seq[File] = + file +: ( + if ( file.isDirectory ) file.listOrFail.flatMap( _.listRecursive ).toVector else Vector[File]() + ) + + def lastModifiedRecursive = listRecursive.map( _.lastModified ).max + + def readAsString = new String( readAllBytes( file.toPath ) ) + + def quote = s"""new _root_.java.io.File(${string.quote})""" + } +} + +trait Module { + def realpath( name: File ) = new File( java.nio.file.Paths.get( name.getAbsolutePath ).normalize.toString ) + def transformFiles( files: Seq[File], transform: String => String ): Seq[File] = { + transformFilesOrError( files, s => Right( transform( s ) ) )._1 + } + + def transformFilesOrError[T]( files: Seq[File], transform: String => Either[T, String] ): ( Seq[File], Seq[( File, T )] ) = { + val results = files.map { file => + val string = file.readAsString + transform( string ).left.map( + file -> _ + ).right.map( + replaced => + if ( string != replaced ) { + val tmpFile = file ++ ".cbt-tmp" + assert( !tmpFile.exists ) + write( tmpFile.toPath, replaced.getBytes ) + move( tmpFile.toPath, file.toPath, REPLACE_EXISTING ) + Some( file ) + } else None + ) + } + + ( results.map( _.right.toOption ).flatten.flatten, results.map( _.left.toOption ).flatten ) + } + + def autoRelative( + files: Seq[File], collector: PartialFunction[( File, String ), String] = { case ( _, r ) => r }, + allowDuplicates: Boolean = false + ): Seq[( File, String )] = { + val map = files.sorted.flatMap { base => + val b = base.getCanonicalFile.string + if ( base.isDirectory ) { + base.listRecursive.map { f => + f -> f.getCanonicalFile.string.stripPrefix( b ).stripPrefix( File.separator ) + } + } else { + Seq( base -> base.getName ) + } + }.collect { + case v @ ( file, _ ) if collector.isDefinedAt( v ) => file -> collector( v ) + } + if ( !allowDuplicates ) { + val relatives = map.unzip._2 + val duplicateFiles = ( relatives diff relatives.distinct ).distinct + assert( + duplicateFiles.isEmpty, { + val rs = relatives.toSet + "Conflicting:\n\n" + + map.filter( rs contains _._2 ).groupBy( _._2 ).mapValues( _.map( _._1 ).sorted ).toSeq.sortBy( _._1 ).map { + case ( name, files ) => s"$name:\n" ++ files.mkString( "\n" ) + }.mkString( "\n\n" ) + } + ) + } + map + } + + /* recursively deletes folders*/ + def deleteRecursive( file: File ): Unit = { + val s = file.string + // some desperate attempts to keep people from accidentally deleting their hard drive + assert( file === file.getCanonicalFile, "deleteRecursive requires previous .getCanonicalFile" ) + assert( file.isAbsolute, "deleteRecursive requires absolute path" ) + assert( file.string != "", "deleteRecursive requires non-empty file path" ) + assert( s.split( File.separator.replace( "\\", "\\\\" ) ).size > 4, "deleteRecursive requires absolute path of at least depth 4" ) + assert( !file.listRecursive.exists( _.isHidden ), "deleteRecursive requires no files to be hidden" ) + assert( file.listRecursive.forall( _.canWrite ), "deleteRecursive requires all files to be writable" ) + if ( file.isDirectory ) { + file.listOrFail.map( deleteRecursive ) + } + file.delete + } +} diff --git a/libraries/interfaces/ExitCode.java b/libraries/interfaces/ExitCode.java new file mode 100644 index 0000000..f29d8c0 --- /dev/null +++ b/libraries/interfaces/ExitCode.java @@ -0,0 +1,4 @@ +package cbt.interfaces; +public interface ExitCode{ + public int integer(); +} diff --git a/libraries/proguard/Proguard.scala b/libraries/proguard/Proguard.scala index 7d57a77..ec5c2e9 100644 --- a/libraries/proguard/Proguard.scala +++ b/libraries/proguard/Proguard.scala @@ -29,8 +29,8 @@ object ProGuard { } case class ProGuard[T]( main: Seq[String] => Int, - T: Seq[File] => T, - log: String => Unit = _ => () + T: Seq[File] => T, + log: String => Unit = _ => () ) { /** @@ -160,9 +160,9 @@ case class ProGuard[T]( private object argsFor { def apply[T: argsFor]( value: T ) = implicitly[argsFor[T]].apply( value ) implicit object SeqFile extends argsFor[Seq[File]]( v => Some( Seq( v.map( _.getPath ).mkString( ":" ) ) ) ) - implicit object File extends argsFor[File]( v => Some( Seq( v.getPath ) ) ) - implicit object String extends argsFor[String]( v => Some( Seq( v ) ) ) - implicit object Int extends argsFor[Int]( i => Some( Seq( i.toString ) ) ) + implicit object File extends argsFor[File]( v => Some( Seq( v.getPath ) ) ) + implicit object String extends argsFor[String]( v => Some( Seq( v ) ) ) + implicit object Int extends argsFor[Int]( i => Some( Seq( i.toString ) ) ) implicit object Boolean extends argsFor[Boolean]( { case false => None diff --git a/libraries/proguard/build/build.scala b/libraries/proguard/build/build.scala index 3ca38b5..754de20 100644 --- a/libraries/proguard/build/build.scala +++ b/libraries/proguard/build/build.scala @@ -1,11 +1,12 @@ package cbt_build.proguard import cbt._ +import cbt_internal._ import java.nio.file.Files._ import java.net._ import java.io._ import scala.xml._ -class Build(val context: Context) extends Scalafmt{ +class Build(val context: Context) extends Library{ def description: String = "Type-safe scala wrapper to interfaces with ProGuard.main runner" def inceptionYear = 2017 @@ -14,11 +15,6 @@ class Build(val context: Context) extends Scalafmt{ compile } - override def scalafmt = super.scalafmt.copy( - config = Scalafmt.cbtRecommendedConfig, - whiteSpaceInParenthesis = true - ) - override def compile = { // currently suffers from non-deterministic formatting. Try a few times to reproduce commit state. val formatted = scalafmt.apply.map(_.string).mkString("\n") diff --git a/libraries/proguard/build/build/build.scala b/libraries/proguard/build/build/build.scala index 7928cfa..5057404 100644 --- a/libraries/proguard/build/build/build.scala +++ b/libraries/proguard/build/build/build.scala @@ -1,13 +1,11 @@ -package cbt_build.proguard.build +package proguard_build.build import cbt._ -class Build(val context: Context) extends BuildBuild{ +class Build(val context: Context) extends BuildBuild with CbtInternal{ override def dependencies = ( super.dependencies ++ // don't forget super.dependencies here for scala-library, etc. Resolver( mavenCentral, sonatypeReleases ).bind( ScalaDependency("org.scala-lang.modules","scala-xml","1.0.5"), "org.ccil.cowan.tagsoup" % "tagsoup" % "1.2.1" - ) ++ Seq( - plugins.scalafmt - ) + ) ++ Seq( cbtInternal.library ) ) } diff --git a/libraries/reflect/StaticMethod.scala b/libraries/reflect/StaticMethod.scala new file mode 100644 index 0000000..e2a0d07 --- /dev/null +++ b/libraries/reflect/StaticMethod.scala @@ -0,0 +1,4 @@ +package cbt.reflect +case class StaticMethod[Arg, Result]( function: Arg => Result, name: String ) extends ( Arg => Result ) { + def apply( arg: Arg ): Result = function( arg ) +} diff --git a/libraries/reflect/build/build.scala b/libraries/reflect/build/build.scala new file mode 100644 index 0000000..5c27090 --- /dev/null +++ b/libraries/reflect/build/build.scala @@ -0,0 +1,8 @@ +package cbt_build.reflect +import cbt._ +import cbt_internal._ +class Build(val context: Context) extends Library{ + override def inceptionYear = 2017 + override def description = "discover classes on your classpath and invoke methods reflectively, preventing System.exit" + override def dependencies = super.dependencies :+ libraries.file +} diff --git a/libraries/reflect/build/build/build.scala b/libraries/reflect/build/build/build.scala new file mode 100644 index 0000000..d3f98ce --- /dev/null +++ b/libraries/reflect/build/build/build.scala @@ -0,0 +1,5 @@ +package cbt_build.reflect.build +import cbt._ +class Build(val context: Context) extends BuildBuild with CbtInternal{ + override def dependencies = super.dependencies :+ cbtInternal.library +} diff --git a/libraries/reflect/reflect.scala b/libraries/reflect/reflect.scala new file mode 100644 index 0000000..c18d926 --- /dev/null +++ b/libraries/reflect/reflect.scala @@ -0,0 +1,177 @@ +package cbt.reflect + +import java.io.File +import java.lang.reflect.{ Constructor, Method, InvocationTargetException, Modifier } + +import scala.reflect.ClassTag + +import cbt.ExitCode +import cbt.file._ + +object `package` extends Module { + implicit class CbtClassOps( val c: Class[_] ) extends AnyVal with ops.CbtClassOps + implicit class CbtConstructorOps( val c: Constructor[_] ) extends AnyVal with ops.CbtConstructorOps + implicit class CbtMethodOps( val m: Method ) extends AnyVal with ops.CbtMethodOps +} + +package ops { + trait CbtClassOps extends Any { + def c: Class[_] + def name = c.getName + def method( name: String, parameterTypes: Class[_]* ) = c.getMethod( name, parameterTypes: _* ) + def methods = c.getMethods + def modifiers = c.getModifiers + def constructors = c.getConstructors + def declaredMethods = c.getDeclaredMethods + def isInterface = Modifier.isInterface( c.getModifiers ) + def isAbstract = Modifier.isAbstract( c.getModifiers ) + def isPrivate = Modifier.isPrivate( c.getModifiers ) + def isProtected = Modifier.isProtected( c.getModifiers ) + def isPublic = Modifier.isPublic( c.getModifiers ) + def isFinal = Modifier.isFinal( c.getModifiers ) + } + trait CbtConstructorOps extends Any { + def c: Constructor[_] + def parameterTypes = c.getParameterTypes + } + trait CbtMethodOps extends Any { + def m: Method + def name = m.getName + def declaringClass = m.getDeclaringClass + def modifiers = m.getModifiers + def parameters = m.getParameters + def parameterTypes = m.getParameterTypes + def returnType = m.getReturnType + + def isAbstract = Modifier.isAbstract( m.getModifiers ) + def isFinal = Modifier.isFinal( m.getModifiers ) + def isNative = Modifier.isNative( m.getModifiers ) + def isPrivate = Modifier.isPrivate( m.getModifiers ) + def isProtected = Modifier.isProtected( m.getModifiers ) + def isPublic = Modifier.isPublic( m.getModifiers ) + def isStatic = Modifier.isStatic( m.getModifiers ) + def isStrict = Modifier.isStrict( m.getModifiers ) + def isSynchronized = Modifier.isSynchronized( m.getModifiers ) + def isTransient = Modifier.isTransient( m.getModifiers ) + def isVolatile = Modifier.isVolatile( m.getModifiers ) + } +} +trait Module { + def runMain( cls: Class[_], args: Seq[String] ): ExitCode = + discoverStaticExitMethodForced[Array[String]]( cls, "main" ).apply( args.to ) + + def discoverMain( cls: Class[_] ): Option[StaticMethod[Seq[String], ExitCode]] = { + discoverStaticExitMethod[Array[String]]( cls, "main" ) + .map( f => + f.copy( + function = ( arg: Seq[String] ) => f.function( arg.to ) + ) ) + } + + /** ignoreMissingClasses allows ignoring other classes root directories which are subdirectories of this one */ + def iterateClasses( + classesRootDirectory: File, + classLoader: ClassLoader, + ignoreMissingClasses: Boolean + ): Seq[Class[_]] = + iterateClassNames( classesRootDirectory ) + .map { name => + try { + classLoader.loadClass( name ) + } catch { + case e: ClassNotFoundException if ignoreMissingClasses => null + case e: NoClassDefFoundError if ignoreMissingClasses => null + } + } + .filterNot( ignoreMissingClasses && _ == null ) + + /** Given a directory corresponding to the root package, iterate + * the names of all classes derived from the class files found + */ + def iterateClassNames( classesRootDirectory: File ): Seq[String] = + classesRootDirectory.listRecursive + .filter( _.isFile ) + .map( _.getPath ) + .collect { + // no $ to avoid inner classes + case path if !path.contains( "$" ) && path.endsWith( ".class" ) => + path + .stripSuffix( ".class" ) + .stripPrefix( classesRootDirectory.getPath ) + .stripPrefix( File.separator ) // 1 for the slash + .replace( File.separator, "." ) + } + + def discoverStaticExitMethodForced[Arg: ClassTag]( + cls: Class[_], name: String + ): StaticMethod[Arg, ExitCode] = { + val f = discoverStaticMethodForced[Arg, Unit]( cls, name ) + f.copy( + function = arg => trapExitCode { f.function( arg ); ExitCode.Success } + ) + } + + def discoverStaticMethodForced[Arg, Result]( + cls: Class[_], name: String + )( + implicit + Result: ClassTag[Result], Arg: ClassTag[Arg] + ): StaticMethod[Arg, Result] = { + val m = cls.method( name, Arg.runtimeClass ) + assert( Result.runtimeClass.isAssignableFrom( m.returnType ) ) + typeStaticMethod( m ) + } + + def discoverStaticExitMethod[Arg: ClassTag]( + cls: Class[_], name: String + ): Option[StaticMethod[Arg, ExitCode]] = + discoverStaticMethod[Arg, Unit]( cls, name ).map( f => + f.copy( + function = arg => trapExitCode { f.function( arg ); ExitCode.Success } + ) ) + + def discoverStaticMethod[Arg, Result]( + cls: Class[_], name: String + )( + implicit + Result: ClassTag[Result], Arg: ClassTag[Arg] + ): Option[StaticMethod[Arg, Result]] = { + Some( cls ) + .filterNot( _.isAbstract ) + .filterNot( _.isInterface ) + .flatMap( _ + .getMethods + .find( m => + !m.isAbstract + && m.isPublic + && m.name == name + && m.parameterTypes.toList == List( Arg.runtimeClass ) + && Result.runtimeClass.isAssignableFrom( m.returnType ) ) ) + .map( typeStaticMethod ) + } + + def typeStaticMethod[Arg, Result]( method: Method ): StaticMethod[Arg, Result] = { + val m = method + val instance = + if ( m.isStatic ) null + else m.declaringClass.newInstance // Dottydoc needs this. It's main method is not static. + StaticMethod( + arg => m.invoke( instance, arg.asInstanceOf[AnyRef] ).asInstanceOf[Result], + m.getClass.name.stripSuffix( "$" ) ++ "." ++ m.name ++ "( " + ++ m.parameters.map( _.getType.name ).mkString( ", " ) + ++ " )" + ) + } + + def trapExitCodeOrValue[T]( result: => T, i: Int = 5 ): Either[ExitCode, T] = { + TrapSystemExit.run( + new TrapSystemExit[Either[ExitCode, T]] { + def run = Right( result ) + def wrap( exitCode: Int ) = Left( new ExitCode( exitCode ) ) + } + ) + } + + def trapExitCode( code: => ExitCode ): ExitCode = + trapExitCodeOrValue( code ).merge +} diff --git a/nailgun_launcher/NailgunLauncher.java b/nailgun_launcher/NailgunLauncher.java index fe9f27f..1c6f3b5 100644 --- a/nailgun_launcher/NailgunLauncher.java +++ b/nailgun_launcher/NailgunLauncher.java @@ -5,6 +5,7 @@ import java.net.*; import java.security.*; import java.util.*; import static cbt.Stage0Lib.*; +import cbt.reflect.TrapSystemExit; import static java.io.File.pathSeparator; import java.nio.file.*; import static java.nio.file.Files.write; @@ -66,7 +67,7 @@ public class NailgunLauncher{ return; } - System.setSecurityManager( new TrapSecurityManager() ); + System.setSecurityManager( TrapSystemExit.createSecurityManager(initialSecurityManager) ); installProxySettings(); String[] diff = args[0].split("\\."); long start = _start - (Long.parseLong(diff[0]) * 1000L) - Long.parseLong(diff[1]); @@ -146,7 +147,7 @@ public class NailgunLauncher{ String nailgunTarget = nailgunSources + TARGET; String stage1Sources = cbtHome + "/" + STAGE1; String stage1Target = stage1Sources + TARGET; - File compatibilitySources = new File(cbtHome + "/compatibility"); + String compatibilitySources = cbtHome + "/compatibility"; String mavenCache = cache + "maven"; String mavenUrl = "https://repo1.maven.org/maven2"; File loopFile = new File(cwd + "/target/.cbt-loop.tmp"); @@ -171,9 +172,14 @@ public class NailgunLauncher{ compatibilityLastModified = new File( compatibilityTarget + "../classes.last-success" ).lastModified(); } else { compatibilitySourceFiles = new ArrayList(); - for( File f: compatibilitySources.listFiles() ){ - if( f.isFile() && f.toString().endsWith(".java") ){ - compatibilitySourceFiles.add(f); + for( String d: new String[]{ + compatibilitySources, + cbtHome + "/libraries/interfaces" + } ){ + for( File f: new File(d).listFiles() ){ + if( f.isFile() && f.toString().endsWith(".java") ){ + compatibilitySourceFiles.add(f); + } } } @@ -206,9 +212,16 @@ public class NailgunLauncher{ String stage1Classpath = classpath( stage1ClasspathArray ); stage1SourceFiles = new ArrayList(); - for( File f: new File(stage1Sources).listFiles() ){ - if( f.isFile() && f.toString().endsWith(".scala") ){ - stage1SourceFiles.add(f); + for( String d: new String[]{ + stage1Sources, + cbtHome + "/libraries/reflect", + cbtHome + "/libraries/common-1", + cbtHome + "/libraries/file" + } ){ + for( File f: new File(d).listFiles() ){ + if( f.isFile() && (f.toString().endsWith(".scala") || f.toString().endsWith(".java")) ){ + stage1SourceFiles.add(f); + } } } diff --git a/nailgun_launcher/ProxySecurityManager.java b/nailgun_launcher/ProxySecurityManager.java deleted file mode 100644 index 1a6e49c..0000000 --- a/nailgun_launcher/ProxySecurityManager.java +++ /dev/null @@ -1,102 +0,0 @@ -package cbt; - -import java.security.*; -import java.io.FileDescriptor; -import java.net.InetAddress; - -/* -SecurityManager proxy that forwards all calls to the provided target if != null. -Useful to replace a previously installed SecurityManager, overriding some methods -but forwarding the rest. -*/ -public class ProxySecurityManager extends SecurityManager{ - private SecurityManager target; - public ProxySecurityManager(SecurityManager target){ - this.target = target; - } - public Object getSecurityContext() { - if(target != null) - return target.getSecurityContext(); - else return super.getSecurityContext(); - } - public void checkPermission(Permission perm) { - if(target != null) target.checkPermission(perm); - } - public void checkPermission(Permission perm, Object context) { - if(target != null) target.checkPermission(perm, context); - } - public void checkCreateClassLoader() { - if(target != null) target.checkCreateClassLoader(); - } - public void checkAccess(Thread t) { - if(target != null) target.checkAccess(t); - } - public void checkAccess(ThreadGroup g) { - if(target != null) target.checkAccess(g); - } - public void checkExit(int status) { - if(target != null) target.checkExit(status); - } - public void checkExec(String cmd) { - if(target != null) target.checkExec(cmd); - } - public void checkLink(String lib) { - if(target != null) target.checkLink(lib); - } - public void checkRead(FileDescriptor fd) { - if(target != null) target.checkRead(fd); - } - public void checkRead(String file) { - if(target != null) target.checkRead(file); - } - public void checkRead(String file, Object context) { - if(target != null) target.checkRead(file, context); - } - public void checkWrite(FileDescriptor fd) { - if(target != null) target.checkWrite(fd); - } - public void checkWrite(String file) { - if(target != null) target.checkWrite(file); - } - public void checkDelete(String file) { - if(target != null) target.checkDelete(file); - } - public void checkConnect(String host, int port) { - if(target != null) target.checkConnect(host, port); - } - public void checkConnect(String host, int port, Object context) { - if(target != null) target.checkConnect(host, port, context); - } - public void checkListen(int port) { - if(target != null) target.checkListen(port); - } - public void checkAccept(String host, int port) { - if(target != null) target.checkAccept(host, port); - } - public void checkMulticast(InetAddress maddr) { - if(target != null) target.checkMulticast(maddr); - } - public void checkPropertiesAccess() { - if(target != null) target.checkPropertiesAccess(); - } - public void checkPropertyAccess(String key) { - if(target != null) target.checkPropertyAccess(key); - } - public void checkPrintJobAccess() { - if(target != null) target.checkPrintJobAccess(); - } - public void checkPackageAccess(String pkg) { - if(target != null) target.checkPackageAccess(pkg); - } - public void checkPackageDefinition(String pkg) { - if(target != null) target.checkPackageDefinition(pkg); - } - public void checkSetFactory() { - if(target != null) target.checkSetFactory(); - } - public ThreadGroup getThreadGroup() { - if(target != null) - return target.getThreadGroup(); - else return super.getThreadGroup(); - } -} diff --git a/nailgun_launcher/Stage0Lib.java b/nailgun_launcher/Stage0Lib.java index 8ab6150..34af7b0 100644 --- a/nailgun_launcher/Stage0Lib.java +++ b/nailgun_launcher/Stage0Lib.java @@ -12,6 +12,7 @@ import static cbt.NailgunLauncher.*; import java.nio.file.*; import java.nio.file.attribute.FileTime; import static java.lang.Math.min; +import cbt.reflect.TrapSystemExit; public class Stage0Lib{ public static void _assert(boolean condition, Object msg){ @@ -21,22 +22,21 @@ public class Stage0Lib{ } public static int runMain(String cls, String[] args, ClassLoader cl) throws Throwable{ - boolean trapExitCodeBefore = TrapSecurityManager.trapExitCode().get(); - try{ - TrapSecurityManager.trapExitCode().set(true); - cl.loadClass(cls) - .getMethod("main", String[].class) - .invoke( null, (Object) args); - return 0; - }catch( InvocationTargetException exception ){ - Throwable cause = exception.getCause(); - if(TrapSecurityManager.isTrappedExit(cause)){ - return TrapSecurityManager.exitCode(cause); + return TrapSystemExit.run( + new TrapSystemExit(){ + @Override + public Integer run() throws Throwable{ + cl.loadClass(cls) + .getMethod("main", String[].class) + .invoke( null, (Object) args); + return 0; + } + @Override + public Integer wrap(int exitCode){ + return exitCode; + } } - throw exception; - } finally { - TrapSecurityManager.trapExitCode().set(trapExitCodeBefore); - } + ); } public static Object get(Object object, String method) throws Throwable{ @@ -111,7 +111,8 @@ public class Stage0Lib{ "-d", target, "-S-deprecation", "-S-feature", - "-S-unchecked" + "-S-unchecked", + "-S-language:existentials" } ) ); diff --git a/nailgun_launcher/TrapSecurityManager.java b/nailgun_launcher/TrapSecurityManager.java deleted file mode 100644 index be59671..0000000 --- a/nailgun_launcher/TrapSecurityManager.java +++ /dev/null @@ -1,83 +0,0 @@ -package cbt; -import java.security.*; -/* -When enabled, this SecurityManager turns System.exit(...) calls into exceptions that can be caught and handled. -Installing a SecurityManager is a global side-effect and thus needs extra care in a persistent -background process like CBT's. The current approach is install it once during JVM-startup. -When disabled this delegates to the SecurityManager installed before if any, which -would be Nailgun's if running on Nailgun. If we do not delegate to Nailgun, it seems we -could in some cases kill the server process -*/ -public class TrapSecurityManager extends ProxySecurityManager{ - public static ThreadLocal trapExitCode(){ - // storing the flag in the installed security manager - // instead of e.g. a static member is necessary because - // we run multiple versions of CBT with multiple TrapSecurityManager classes - // but we need to affect the installed one - SecurityManager sm = System.getSecurityManager(); - if(sm instanceof TrapSecurityManager){ - return ((TrapSecurityManager) sm)._trapExitCode; - } else { - try{ - @SuppressWarnings("unchecked") - ThreadLocal res = - (ThreadLocal) sm.getClass().getMethod("trapExitCode").invoke(null); - return res; - } catch(Exception e) { - throw new RuntimeException(e); - } - } - } - - private final ThreadLocal _trapExitCode = - new ThreadLocal() { - @Override protected Boolean initialValue() { - return false; - } - }; - - public TrapSecurityManager(){ - super(NailgunLauncher.initialSecurityManager); - } - - public void checkPermission( Permission permission ){ - /* - NOTE: is it actually ok, to just make these empty? - Calling .super leads to ClassNotFound exteption for a lambda. - Calling to the previous SecurityManager leads to a stack overflow - */ - if(!TrapSecurityManager.trapExitCode().get()){ - super.checkPermission(permission); - } - } - public void checkPermission( Permission permission, Object context ){ - /* Does this methods need to be overidden? */ - if(!TrapSecurityManager.trapExitCode().get()){ - super.checkPermission(permission, context); - } - } - - private static final String prefix = "[TrappedExit] "; - - @Override - public void checkExit( int status ){ - if(TrapSecurityManager.trapExitCode().get()){ - // using a RuntimeException and a prefix here instead of a custom - // exception type because this is thrown by the installed TrapSecurityManager - // but other versions of cbt need to be able to catch it, that do not have access - // to that version of the TrapSecurityManager class - throw new RuntimeException(prefix+status); - } - super.checkExit(status); - } - - public static boolean isTrappedExit( Throwable t ){ - return t instanceof RuntimeException && t.getMessage() != null && t.getMessage().startsWith(prefix); - } - - public static int exitCode( Throwable t ){ - assert(isTrappedExit(t)); - return Integer.parseInt( t.getMessage().substring(prefix.length()) ); - } - -} diff --git a/plugins/proguard/Proguard.scala b/plugins/proguard/Proguard.scala index 486d969..f229e66 100644 --- a/plugins/proguard/Proguard.scala +++ b/plugins/proguard/Proguard.scala @@ -24,7 +24,7 @@ object ProGuard { context.logger, transientCache, context.classLoaderCache ).bindOne( MavenDependency(groupId, artifactId, version) - ).runMain(cbt.proguard.ProGuard.mainClass, args: _*).integer, + ).runMain(cbt.proguard.ProGuard.mainClass, args).integer, ClassPath(_), context.logger.log("proguard",_) ) diff --git a/plugins/uber-jar/src/UberJar.scala b/plugins/uber-jar/src/UberJar.scala index f330f83..856b69a 100644 --- a/plugins/uber-jar/src/UberJar.scala +++ b/plugins/uber-jar/src/UberJar.scala @@ -41,7 +41,7 @@ class UberJarLib(lib: cbt.Lib, log: String => Unit) { val uberJar = lib.createJar(jarFile, dirs :+ extracted, mainClass=mainClass) log("Writing uber jar - DONE") - extracted.deleteRecursive + lib.deleteRecursive( extracted ) System.err.println(lib.green("Creating uber jar - DONE")) diff --git a/stage1/Stage1Lib.scala b/stage1/Stage1Lib.scala index 565a06c..8aaa6e6 100644 --- a/stage1/Stage1Lib.scala +++ b/stage1/Stage1Lib.scala @@ -11,31 +11,11 @@ import java.security._ import java.util.{Set=>_,Map=>_,List=>_,_} import javax.xml.bind.annotation.adapters.HexBinaryAdapter -// CLI interop -case class ExitCode(integer: Int){ - def ||( other: => ExitCode ) = if( this == ExitCode.Success ) this else other - def &&( other: => ExitCode ) = if( this != ExitCode.Success ) this else other -} -object ExitCode{ - val Success = ExitCode(0) - val Failure = ExitCode(1) -} - -object CatchTrappedExitCode{ - def unapply(e: Throwable): Option[ExitCode] = { - Option(e) flatMap { - case i: InvocationTargetException => unapply(i.getTargetException) - case e if TrapSecurityManager.isTrappedExit(e) => Some( ExitCode(TrapSecurityManager.exitCode(e)) ) - case _ => None - } - } -} - -class BaseLib{ - def realpath(name: File) = new File(java.nio.file.Paths.get(name.getAbsolutePath).normalize.toString) -} - -class Stage1Lib( logger: Logger ) extends BaseLib{ +class Stage1Lib( logger: Logger ) extends + _root_.cbt.common_1.Module with + _root_.cbt.reflect.Module with + _root_.cbt.file.Module +{ lib => implicit protected val implicitLogger: Logger = logger @@ -99,24 +79,28 @@ class Stage1Lib( logger: Logger ) extends BaseLib{ } } + /* // ========== compilation / execution ========== // TODO: move classLoader first - def runMain( cls: String, args: Seq[String], classLoader: ClassLoader, fakeInstance: Boolean = false ): ExitCode = { + def runMain( className: String, args: Seq[String], classLoader: ClassLoader ): ExitCode = { import java.lang.reflect.Modifier - logger.run(s"Running $cls.main($args) with classLoader: " ++ classLoader.toString) + logger.run(s"Running $className.main($args) with classLoader: " ++ classLoader.toString) trapExitCode{ - val c = classLoader.loadClass(cls) - val m = c.getMethod( "main", classOf[Array[String]] ) - val instance = - if(!fakeInstance) null else c.newInstance - assert( - fakeInstance || (m.getModifiers & java.lang.reflect.Modifier.STATIC) > 0, - "Cannot run non-static method " ++ cls+".main" - ) - m.invoke( instance, args.toArray.asInstanceOf[AnyRef] ) + /* + val cls = classLoader.loadClass(className) + discoverCbtMain( cls ) orElse discoverMain( cls ) getOrElse ( + throw new NoSuchMethodException( "No main method found in " ++ cbt ) + ).apply( arg.toVector )*/ ExitCode.Success } } + */ + + def discoverCbtMainForced( cls: Class[_] ): cbt.reflect.StaticMethod[Context, ExitCode] = + discoverStaticMethodForced[Context, ExitCode]( cls, "cbtMain" ) + + def discoverCbtMain( cls: Class[_] ): Option[cbt.reflect.StaticMethod[Context, ExitCode]] = + discoverStaticMethod[Context, ExitCode]( cls, "cbtMain" ) /** shows an interactive dialogue in the shell asking the user to pick one of many choices */ def pickOne[T]( msg: String, choices: Seq[T] )( show: T => String ): Option[T] = { @@ -149,51 +133,10 @@ class Stage1Lib( logger: Logger ) extends BaseLib{ } /** interactively pick one main class */ - def runClass( mainClasses: Seq[Class[_]] ): Option[Class[_]] = { + def pickClass( mainClasses: Seq[Class[_]] ): Option[Class[_]] = { pickOne( "Which one do you want to run?", mainClasses )( _.toString ) } - /** Given a directory corresponding to the root package, iterate - the names of all classes derived from the class files found */ - def iterateClassNames( classesRootDirectory: File ): Seq[String] = - classesRootDirectory - .listRecursive - .filter(_.isFile) - .map(_.getPath) - .collect{ - // no $ to avoid inner classes - case path if !path.contains("$") && path.endsWith(".class") => - path.stripSuffix(".class") - .stripPrefix(classesRootDirectory.getPath) - .stripPrefix(File.separator) // 1 for the slash - .replace(File.separator, ".") - } - - /** ignoreMissingClasses allows ignoring other classes root directories which are subdirectories of this one */ - def iterateClasses( classesRootDirectory: File, classLoader: ClassLoader, ignoreMissingClasses: Boolean ) = - iterateClassNames(classesRootDirectory).map{ name => - try{ - classLoader.loadClass(name) - } catch { - case e: ClassNotFoundException if ignoreMissingClasses => null - case e: NoClassDefFoundError if ignoreMissingClasses => null - } - }.filterNot(ignoreMissingClasses && _ == null) - - def mainClasses( classesRootDirectory: File, classLoader: ClassLoader ): Seq[Class[_]] = { - val arrayClass = classOf[Array[String]] - val unitClass = classOf[Unit] - - iterateClasses( classesRootDirectory, classLoader, true ).filter( c => - !c.isInterface && - c.getDeclaredMethods().exists( m => - m.getName == "main" - && m.getParameterTypes.toList == List(arrayClass) - && m.getReturnType == unitClass - ) - ) - } - implicit class ClassLoaderExtensions(classLoader: ClassLoader){ def canLoad(className: String) = { try{ @@ -322,8 +265,9 @@ ${sourceFiles.sorted.mkString(" \\\n")} } } } - def redirectOutToErr[T](code: => T): T = { - val ( out, err ) = try{ + + def getOutErr: (ThreadLocal[PrintStream], ThreadLocal[PrintStream]) = + try{ // trying nailgun's System.our/err wrapper val field = System.out.getClass.getDeclaredField("streams") assert(System.out.getClass.getName == "com.martiansoftware.nailgun.ThreadLocalPrintStream") @@ -339,8 +283,8 @@ ${sourceFiles.sorted.mkString(" \\\n")} field.setAccessible(true) val outStream = field.get(System.out) val errStream = field.get(System.err) - assert(outStream.getClass.getName == "cbt.ThreadLocalOutputStream") - assert(errStream.getClass.getName == "cbt.ThreadLocalOutputStream") + assert(outStream.getClass.getName == "cbt.ThreadLocalOutputStream", outStream.getClass.getName) + assert(errStream.getClass.getName == "cbt.ThreadLocalOutputStream", errStream.getClass.getName) val field2 = outStream.getClass.getDeclaredField("threadLocal") field2.setAccessible(true) val out = field2.get(outStream).asInstanceOf[ThreadLocal[PrintStream]] @@ -348,6 +292,8 @@ ${sourceFiles.sorted.mkString(" \\\n")} ( out, err ) } + def redirectOutToErr[T](code: => T): T = { + val ( out, err ) = getOutErr val oldOut: PrintStream = out.get out.set( err.get: PrintStream ) val res = code @@ -355,21 +301,6 @@ ${sourceFiles.sorted.mkString(" \\\n")} res } - def trapExitCodeOrValue[T]( result: => T ): Either[ExitCode,T] = { - val trapExitCodeBefore = TrapSecurityManager.trapExitCode().get - try{ - TrapSecurityManager.trapExitCode().set(true) - Right( result ) - } catch { - case CatchTrappedExitCode(exitCode) => - logger.stage1(s"caught exit code $exitCode") - Left( exitCode ) - } finally { - TrapSecurityManager.trapExitCode().set(trapExitCodeBefore) - } - } - - def trapExitCode( code: => ExitCode ): ExitCode = trapExitCodeOrValue(code).merge def ScalaDependency( groupId: String, artifactId: String, version: String, classifier: Classifier = Classifier.none, diff --git a/stage1/cbt.scala b/stage1/cbt.scala index 8cba9df..64257c2 100644 --- a/stage1/cbt.scala +++ b/stage1/cbt.scala @@ -3,13 +3,19 @@ import java.io._ import java.nio.file._ import java.nio.file.Files._ import java.net._ +import java.lang.reflect._ object `package`{ - implicit class TypeInferenceSafeEquals[T](value: T){ - /** if you don't manually upcast, this will catch comparing different types */ - def ===(other: T) = value == other - def =!=(other: T) = value != other // =!= instead of !==, because it has better precedence - } + implicit class CbtExitCodeOps( val exitCode: ExitCode ) extends AnyVal with common_1.ops.CbtExitCodeOps + implicit class TypeInferenceSafeEquals[T]( val value: T ) extends AnyVal with common_1.ops.TypeInferenceSafeEquals[T] + implicit class CbtBooleanOps( val condition: Boolean ) extends AnyVal with common_1.ops.CbtBooleanOps + implicit class CbtStringOps( val string: String ) extends AnyVal with common_1.ops.CbtStringOps + + implicit class CbtFileOps( val file: File ) extends file.ops.CbtFileOps + + implicit class CbtClassOps( val c: Class[_] ) extends AnyVal with reflect.ops.CbtClassOps + implicit class CbtConstructorOps( val c: Constructor[_] ) extends AnyVal with reflect.ops.CbtConstructorOps + implicit class CbtMethodOps( val m: Method ) extends AnyVal with reflect.ops.CbtMethodOps val mavenCentral = new URL("https://repo1.maven.org/maven2") val jcenter = new URL("https://jcenter.bintray.com") @@ -18,15 +24,6 @@ object `package`{ val sonatypeReleases = sonatypeBase ++ "releases" val sonatypeSnapshots = sonatypeBase ++ "snapshots" - private val lib = new BaseLib - - implicit class CbtBooleanExtensions(condition: Boolean){ - def option[T](value: =>T): Option[T] = if(condition) Some(value) else None - } - implicit class CbtStringExtensions(string: String){ - def escape = string.replace("\\","\\\\").replace("\"","\\\"") - def quote = s""""$escape"""" - } implicit class PathExtensionMethods( path: Path ){ def /(s: String): Path = path.resolve(s) def ++( s: String ): Path = { @@ -36,44 +33,7 @@ object `package`{ Paths.get( path.toString ++ s ) } } - implicit class FileExtensionMethods( file: File ){ - def ++( s: String ): File = { - if(s endsWith "/") throw new Exception( - """Trying to append a String that ends in "/" to a File would loose the trailing "/". Use .stripSuffix("/") if you need to.""" - ) - new File( file.toString ++ s ) - } - def /(s: String): File = new File( file, s ) - def parent = lib.realpath(file ++ "/..") - def string = file.toString - /* recursively deletes folders*/ - def deleteRecursive: Unit = { - val s = file.string - // some desperate attempts to keep people from accidentally deleting their hard drive - assert( file == file.getCanonicalFile, "deleteRecursive requires previous .getCanonicalFile" ) - assert( file.isAbsolute, "deleteRecursive requires absolute path" ) - assert( file.string != "", "deleteRecursive requires non-empty file path" ) - assert( s.split(File.separator.replace("\\","\\\\")).size > 4, "deleteRecursive requires absolute path of at least depth 4" ) - assert( !listRecursive.exists(_.isHidden), "deleteRecursive requires no files to be hidden" ) - assert( listRecursive.forall(_.canWrite), "deleteRecursive requires all files to be writable" ) - if( file.isDirectory ){ - file.listFiles.map(_.deleteRecursive) - } - file.delete - } - - def listOrFail: Seq[File] = Option( file.listFiles ).getOrElse( throw new Exception( "no such file: " + file ) ).toVector - def listRecursive: Seq[File] = { - file +: ( - if( file.isDirectory ) file.listFiles.flatMap(_.listRecursive).toVector else Seq[File]() - ) - } - def lastModifiedRecursive = listRecursive.map(_.lastModified).max - - def readAsString = new String( readAllBytes( file.toPath ) ) - def quote = s"new _root_.java.io.File(${string.quote})" - } implicit class URLExtensionMethods( url: URL ){ def ++( s: String ): URL = new URL( url.toString ++ s ) def show = "/[^/@]+@".r.replaceFirstIn( url.toString, "/" ) // remove credentials when showing url for security reasons diff --git a/stage1/resolver.scala b/stage1/resolver.scala index e3500b3..0f0acaa 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -77,26 +77,25 @@ trait DependencyImplementation extends Dependency{ ) } */ - - def runMain( className: String, args: Seq[String] ) = lib.runMain( className, args, classLoader ) - def flatClassLoader: Boolean = false - def mainClasses: Seq[Class[_]] = exportedClasspath.files.flatMap( lib.mainClasses( _, classLoader ) ) - - def runClass: Option[String] = lib.runClass( mainClasses ).map( _.getName ) + def runMain( className: String, args: Seq[String] ): ExitCode = lib.trapExitCode{ + lib.runMain( classLoader.loadClass( className ), args ) + } - def run( args: String* ): ExitCode = { - runClass.map( runMain( _, args ) ).getOrElse{ - // FIXME: this just doing nothing when class is not found has been repeatedly - // surprising. Let's try to make this more visible than just logging an error. - // Currently blocked on task `recursive` trying every subbuild and would error - // for all that don't have a run class. Maybe that's ok actually. - logger.task( "No main class found for " ++ show ) - ExitCode.Success - } + def runMain( args: Seq[String] ): ExitCode = lib.trapExitCode{ + mainMethod.getOrElse( + throw new RuntimeException( "No main class found in " + this ) + )( args ) } + def mainMethod = lib.pickOne( "Which one do you want to run?", mainMethods )( _.name ) + + def classes = exportedClasspath.files.flatMap( + lib.iterateClasses( _, classLoader, false ) + ) + def mainMethods = classes.flatMap( lib.discoverMain ) + def classLoader: ClassLoader = { if( flatClassLoader ){ new java.net.URLClassLoader(classpath.strings.map(f => new URL("file://" ++ f)).toArray) diff --git a/stage2/BasicBuild.scala b/stage2/BasicBuild.scala index 4158040..ebb6a40 100644 --- a/stage2/BasicBuild.scala +++ b/stage2/BasicBuild.scala @@ -13,12 +13,7 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with SbtDep def moduleKey: String = "BaseBuild("+target.string+")" implicit def transientCache: java.util.Map[AnyRef,AnyRef] = context.transientCache - object libraries{ - private def dep(name: String) = DirectoryDependency( context.cbtHome / "libraries" / name ) - def captureArgs = dep( "capture_args" ) - def eval = dep( "eval" ) - def proguard = dep( "proguard" ) - } + implicit def libraries(implicit context: Context): libraries = new libraries(context) // library available to builds implicit protected final val logger: Logger = context.logger @@ -211,7 +206,8 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with SbtDep ) } - def run: ExitCode = run( context.args: _* ) + def run: ExitCode = runMain( context.args ) + def test: Dependency = { val testDirectory = projectDirectory / "test" if( (testDirectory / lib.buildDirectoryName / lib.buildFileName).exists ){ @@ -225,6 +221,7 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with SbtDep } } } + def t: Any = lib.callReflective( test, Some("run"), context ) def rt = recursiveUnsafe(Some("test.run")) diff --git a/stage2/BuildBuild.scala b/stage2/BuildBuild.scala index 17ccb36..cc69905 100644 --- a/stage2/BuildBuild.scala +++ b/stage2/BuildBuild.scala @@ -3,22 +3,6 @@ import java.nio.file._ import java.io.File class ConcreteBuildBuild(val context: Context) extends BuildBuild -class plugins(implicit context: Context){ - // TODO: move this out of the OO - private def plugin(dir: String) = cbt.DirectoryDependency(context.cbtHome / "plugins" / dir) - final lazy val googleJavaFormat = plugin( "google-java-format" ) - final lazy val proguard = plugin( "proguard" ) - final lazy val sbtLayout = plugin( "sbt_layout" ) - final lazy val scalafmt = plugin( "scalafmt" ) - final lazy val scalaJs = plugin( "scalajs" ) - final lazy val scalapb = plugin( "scalapb" ) - final lazy val scalariform = plugin( "scalariform" ) - final lazy val scalaTest = plugin( "scalatest" ) - final lazy val sonatypeRelease = plugin( "sonatype-release" ) - final lazy val uberJar = plugin( "uber-jar" ) - final lazy val wartremover = plugin( "wartremover" ) - final lazy val scalafix = plugin( "scalafix" ) -} trait BuildBuild extends BaseBuild{ override def dependencies = super.dependencies :+ context.cbtDependency @@ -31,7 +15,7 @@ trait BuildBuild extends BaseBuild{ ) } -trait CbtInternal extends BuildBuild{ +trait CbtInternal extends BaseBuild{ protected object cbtInternal{ def shared = DirectoryDependency(context.cbtHome / "/internal/plugins/shared") def library = DirectoryDependency(context.cbtHome / "/internal/plugins/library") diff --git a/stage2/DirectoryDependency.scala b/stage2/DirectoryDependency.scala index cfc0bfd..9b07702 100644 --- a/stage2/DirectoryDependency.scala +++ b/stage2/DirectoryDependency.scala @@ -96,7 +96,7 @@ object DirectoryDependency { ) } else { val buildClass = buildClasses.head - buildClass.getConstructors.find( _.getParameterTypes.toList === List( classOf[Context] ) ).map { + buildClass.constructors.find( _.parameterTypes.toList === List( classOf[Context] ) ).map { _.newInstance( managedContext ).asInstanceOf[AnyRef] }.getOrElse { throw new Exception( diff --git a/stage2/Lib.scala b/stage2/Lib.scala index fd3346e..6488c1a 100644 --- a/stage2/Lib.scala +++ b/stage2/Lib.scala @@ -64,10 +64,8 @@ final class Lib(val logger: Logger) extends Stage1Lib(logger){ c => c .getMethods - .filter{ m => - java.lang.reflect.Modifier.isPublic(m.getModifiers) - } - .filter( _.getParameterTypes.length == 0 ) + .filter( _.isPublic ) + .filter( _.parameterTypes.length == 0 ) .map(m => NameTransformer.decode(m.getName) -> m) .filterNot(_._1 contains "$") ).toMap @@ -262,34 +260,6 @@ final class Lib(val logger: Logger) extends Stage1Lib(logger){ m } - def autoRelative( files: Seq[File], collector: PartialFunction[(File,String), String] = { case (_,r) => r }): Seq[(File, String)] = { - val map = files.sorted.flatMap{ base => - val b = base.getCanonicalFile.string - if( base.isDirectory ){ - base.listRecursive.map{ f => - f -> f.getCanonicalFile.string.stripPrefix(b).stripPrefix(File.separator) - } - } else { - Seq( base -> base.getName ) - } - }.collect{ - case v@(file, _) if collector.isDefinedAt(v) => file -> collector(v) - } - val relatives = map.unzip._2 - val duplicateFiles = (relatives diff relatives.distinct).distinct - assert( - duplicateFiles.isEmpty, - { - val rs = relatives.toSet - "Conflicting:\n\n" + - map.filter(rs contains _._2).groupBy(_._2).mapValues(_.map(_._1).sorted).toSeq.sortBy(_._1).map{ - case (name, files) => s"$name:\n" ++ files.mkString("\n") - }.mkString("\n\n") - } - ) - map - } - def createJar( jarFile: File, files: Seq[File], mainClass: Option[String] = None ): Option[File] = { deleteIfExists(jarFile.toPath) if( files.isEmpty ){ @@ -474,29 +444,5 @@ final class Lib(val logger: Logger) extends Stage1Lib(logger){ ) findOuterMostModuleDirectory(directory.getParentFile) else directory } - def transformFiles( files: Seq[File], transform: String => String ): Seq[File] = { - transformFilesOrError( files, s => Right(transform(s)) )._1 - } - - def transformFilesOrError[T]( files: Seq[File], transform: String => Either[T,String] ): ( Seq[File], Seq[(File, T)] ) = { - val results = files.map{ file => - val string = file.readAsString - transform( string ).left.map( - file -> _ - ).right.map( - replaced => - if( string != replaced ) { - val tmpFile = file ++ ".cbt-tmp" - assert( !tmpFile.exists ) - write( tmpFile, replaced ) - move( tmpFile.toPath, file.toPath, StandardCopyOption.REPLACE_EXISTING ) - Some( file ) - } else None - ) - } - - ( results.map(_.right.toOption).flatten.flatten, results.map(_.left.toOption).flatten ) - } - def clearScreen = System.err.println( (27.toChar +: "[2J").mkString ) } diff --git a/stage2/libraries.scala b/stage2/libraries.scala new file mode 100644 index 0000000..3d2951c --- /dev/null +++ b/stage2/libraries.scala @@ -0,0 +1,12 @@ +package cbt +class libraries( context: Context ) { + private def dep( name: String ) = DirectoryDependency( context.cbtHome / "libraries" / name )( context ) + def captureArgs = dep( "capture_args" ) + def eval = dep( "eval" ) + def file = dep( "file" ) + def proguard = dep( "proguard" ) + def reflect = dep( "reflect" ) + def common_0 = dep( "common-0" ) + def common_1 = dep( "common-1" ) + def interfaces = dep( "interfaces" ) +} diff --git a/stage2/plugins.scala b/stage2/plugins.scala new file mode 100644 index 0000000..eca28f0 --- /dev/null +++ b/stage2/plugins.scala @@ -0,0 +1,17 @@ +package cbt +class plugins( implicit context: Context ) { + // TODO: move this out of the OO + private def plugin( dir: String ) = DirectoryDependency( context.cbtHome / "plugins" / dir ) + final lazy val googleJavaFormat = plugin( "google-java-format" ) + final lazy val proguard = plugin( "proguard" ) + final lazy val sbtLayout = plugin( "sbt_layout" ) + final lazy val scalafix = plugin( "scalafix" ) + final lazy val scalafmt = plugin( "scalafmt" ) + final lazy val scalaJs = plugin( "scalajs" ) + final lazy val scalapb = plugin( "scalapb" ) + final lazy val scalariform = plugin( "scalariform" ) + final lazy val scalaTest = plugin( "scalatest" ) + final lazy val sonatypeRelease = plugin( "sonatype-release" ) + final lazy val uberJar = plugin( "uber-jar" ) + final lazy val wartremover = plugin( "wartremover" ) +} diff --git a/stage2/plugins/Dotty.scala b/stage2/plugins/Dotty.scala index 0bbaf44..766e9d1 100644 --- a/stage2/plugins/Dotty.scala +++ b/stage2/plugins/Dotty.scala @@ -107,11 +107,9 @@ class DottyLib( ) ++ compileArgs ++ sourceFiles.map(_.toString) logger.lib("creating docs for source files "+args.mkString(", ")) val exitCode = redirectOutToErr{ - runMain( + dottyCompiler.runMain( "dotty.tools.dottydoc.DocDriver", - args, - dottyCompiler.classLoader, - fakeInstance = true // this is a hack as Dottydoc's main method is not static + args ) } System.err.println("done") diff --git a/test/build/build.scala b/test/build/build.scala index 2777511..36261b5 100644 --- a/test/build/build.scala +++ b/test/build/build.scala @@ -1,3 +1,4 @@ +package cbt_build.cbt.test import cbt._ class Build(val context: cbt.Context) extends BaseBuild{ override def dependencies = super.dependencies :+ context.cbtDependency -- cgit v1.2.3 From 9af839c58eec4a6e64a5c200c3ddea6458998916 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Mon, 27 Mar 2017 23:52:09 -0400 Subject: fix nailgun under ubuntu and limit memory so circle is ok with 2 running which we need for forked tests --- cbt | 54 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/cbt b/cbt index 61fc58a..6e32e50 100755 --- a/cbt +++ b/cbt @@ -76,14 +76,6 @@ fi # exit 1 # fi -NG_EXECUTABLE=$(which ng || which ng-nailgun) -NG_SERVER_JAR=$(find /usr/local/Cellar/nailgun/*/libexec/nailgun-server-*.jar 2>/dev/null || find /usr/share/java/nailgun-server.jar 2>/dev/null) -NG_SERVER=$(which ng-server || echo "java -jar $NG_SERVER_JAR") -nailgun_installed=0 -if [ "$NG_EXECUTABLE" == "" ] || [ "$NG_SERVER" == "" ]; then - nailgun_installed=1 - echo "(Note: nailgun not found. It makes CBT faster! Try 'brew install nailgun' or 'apt install nailgun'.)" 1>&2 -fi which realpath >/dev/null 2>/dev/null realpath_installed=$? which gcc >/dev/null 2>/dev/null @@ -99,9 +91,6 @@ if [ ! $gpg_installed -eq 0 ]; then echo "(Note: gpg not found. In order to use publishSigned you'll need it.)" 1>&2 fi -NAILGUN_PORT=4444 -NG="$NG_EXECUTABLE --nailgun-port $NAILGUN_PORT" - CWD=$(pwd) CBT_SCRIPT="$(readlink "$0")" @@ -123,8 +112,9 @@ export NAILGUN="$CBT_HOME"/nailgun_launcher/ export TARGET=target/scala-2.11/classes/ mkdir -p "$NAILGUN$TARGET" -nailgun_out=$NAILGUN/target/nailgun.stdout.log -nailgun_err=$NAILGUN/target/nailgun.strerr.log +nailgun_out="$NAILGUN/target/nailgun.stdout.log" +nailgun_err="$NAILGUN/target/nailgun.strerr.log" + DEBUG="" foo(){ while test $# -gt 0; do @@ -147,9 +137,33 @@ foo(){ foo "$@" +JAVA_OPTS_CBT=($DEBUG -Xmx1536m -Xss10M -XX:MaxJavaStackTraceDepth=-1 -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -Xverify:none) + +# ng on osx, install via brew install nailgun +# ng-nailgun on unbuntu, install via apt-get install nailgun +NG_EXECUTABLE=$(which ng || env which ng-nailgun) + +# on osx, install via brew install nailgun, /usr/local/Cellar/nailgun/*/libexec/nailgun-server-*.jar +# on unbuntu, install via apt-get install nailgun, /usr/share/java/nailgun.jar +# on debian, install via apt-get install nailgun, /usr/share/java/nailgun-server.jar +NG_SERVER_JAR=$( \ + find /usr/local/Cellar/nailgun/*/libexec/nailgun-server-*.jar 2>/dev/null \ + || find /usr/share/java/nailgun.jar 2>/dev/null \ + || find /usr/share/java/nailgun-server.jar 2>/dev/null \ +) + +nailgun_installed=0 +if [ "$NG_EXECUTABLE" == "" ] || [ "$NG_SERVER_JAR" == "" ]; then + nailgun_installed=1 + echo "(Note: nailgun not found. It makes CBT faster! Try 'brew install nailgun' or 'apt-get install nailgun'.)" 1>&2 +fi + +NAILGUN_PORT=4444 +NG="$NG_EXECUTABLE --nailgun-port $NAILGUN_PORT" + if [ "$1" = "kill" ]; then echo "Stopping background process (nailgun)" 1>&2 - $NG ng-stop >> $nailgun_out 2>> $nailgun_err & + $NG ng-stop >> "$nailgun_out" 2>> "$nailgun_err" & exit 1 fi @@ -189,7 +203,7 @@ fi if [ $use_nailgun -eq 0 ] && [ ! $server_up -eq 0 ]; then log "Starting background process (nailgun)" "$@" # try to start nailgun-server, just in case it's not up - $NG_SERVER 127.0.0.1:$NAILGUN_PORT >> $nailgun_out 2>> $nailgun_err & + java "${options[@]}" "${JAVA_OPTS_CBT[@]}" -jar "$NG_SERVER_JAR" 127.0.0.1:$NAILGUN_PORT >> "$nailgun_out" 2>> "$nailgun_err" & fi stage1 () { @@ -203,7 +217,7 @@ stage1 () { exitCode=0 if [ $changed -eq 0 ]; then echo "Stopping background process (nailgun) if running" 1>&2 - $NG ng-stop >> $nailgun_out 2>> $nailgun_err & + $NG ng-stop >> "$nailgun_out" 2>> "$nailgun_err" & #rm $NAILGUN$TARGET/cbt/*.class 2>/dev/null # defensive delete of potentially broken class files echo "Compiling cbt/nailgun_launcher" 1>&2 COMPILE_TIME=$(date +%YY%mm%dd%HH%MM.%SS|sed "s/[YmdHMS]//g") @@ -214,7 +228,7 @@ stage1 () { touch -t "$COMPILE_TIME" "$NAILGUN_INDICATOR" if [ $use_nailgun -eq 0 ]; then echo "Starting background process (nailgun)" 1>&2 - ng-server 127.0.0.1:$NAILGUN_PORT >> $nailgun_out 2>> $nailgun_err & + java "${options[@]}" "${JAVA_OPTS_CBT[@]}" -jar "$NG_SERVER_JAR" 127.0.0.1:$NAILGUN_PORT >> "$nailgun_out" 2>> "$nailgun_err" & sleep 1 fi fi @@ -228,15 +242,15 @@ stage1 () { log "Running JVM directly" "$@" options=($JAVA_OPTS) # JVM options to improve startup time. See https://github.com/cvogt/cbt/pull/262 - java "${options[@]}" $DEBUG -Xmx3072m -Xss10M -XX:MaxJavaStackTraceDepth=-1 -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -Xverify:none -cp "$NAILGUN$TARGET" cbt.NailgunLauncher "$(time_taken)" "$CWD" "$loop" "$@" + java "${options[@]}" "${JAVA_OPTS_CBT[@]}" -cp "$NAILGUN$TARGET" cbt.NailgunLauncher "$(time_taken)" "$CWD" "$loop" "$@" exitCode=$? else log "Running via background process (nailgun)" "$@" for i in 0 1 2 3 4 5 6 7 8 9; do log "Adding classpath." "$@" - $NG ng-cp "$NAILGUN$TARGET" >> $nailgun_out 2>> $nailgun_err + $NG ng-cp "$NAILGUN$TARGET" >> "$nailgun_out" 2>> "$nailgun_err" log "Checking if nailgun is up yet." "$@" - $NG cbt.NailgunLauncher check-alive >> $nailgun_out 2>> $nailgun_err + $NG cbt.NailgunLauncher check-alive >> "$nailgun_out" 2>> "$nailgun_err" alive=$? if [ $alive -eq 131 ] || [ $alive -eq 33 ]; then # the 33 is not working right now -- cgit v1.2.3 From 1bf88c580aedd82dc7e3ebede2ba10e3fecf298e Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Mon, 20 Mar 2017 22:41:02 -0400 Subject: allow running tests in the same process, forked or forked in direct mode --- circle.yml | 27 +++++++++--- test/build/build.scala | 3 ++ test/test.scala | 114 ++++++++++++++++++++++++++++++------------------- 3 files changed, 94 insertions(+), 50 deletions(-) diff --git a/circle.yml b/circle.yml index e9e634c..467c72d 100644 --- a/circle.yml +++ b/circle.yml @@ -3,25 +3,38 @@ machine: version: oraclejdk8 dependencies: -# cache_directories: -# - "cache" + cache_directories: + - "~/cache" override: - - sudo apt-add-repository "deb http://archive.ubuntu.com/ubuntu trusty-backports main restricted universe" - - sudo apt-get update - - sudo apt-get install -t trusty-backports shellcheck + - mkdir -p ~/cache + - wget http://mirrors.kernel.org/ubuntu/pool/universe/n/nailgun/nailgun_0.9.0+trunk95-3_amd64.deb --continue -O ~/cache/nailgun_0.9.0+trunk95-3_amd64.deb + - sudo dpkg -i ~/cache/nailgun_0.9.0+trunk95-3_amd64.deb + - wget http://mirrors.kernel.org/ubuntu/pool/universe/s/shellcheck/shellcheck_0.3.3-1~ubuntu14.04.1_amd64.deb --continue -O ~/cache/shellcheck_0.3.3-1~ubuntu14.04.1_amd64.deb + - sudo dpkg -i ~/cache/shellcheck_0.3.3-1~ubuntu14.04.1_amd64.deb compile: override: + - ./cbt direct compile + - ./cbt direct test.compile - ./cbt compile - - git diff --exit-code - - ./cbt direct - ./cbt -Dlog=all + - git diff --exit-code test: override: - rm ~/.gitconfig # avoid url replacement breaking jgit - ./cbt direct test.run slow: timeout: 1800 + - ./cbt direct test.run slow: + timeout: 1800 + - ./cbt direct test.run slow fork: + timeout: 1800 + - ./cbt direct test.run slow fork direct: + timeout: 1800 - ./cbt test.run slow: timeout: 1800 + - ./cbt test.run slow fork: + timeout: 1800 + - ./cbt test.run slow fork direct: + timeout: 1800 - git diff --exit-code diff --git a/test/build/build.scala b/test/build/build.scala index 36261b5..792b34d 100644 --- a/test/build/build.scala +++ b/test/build/build.scala @@ -3,5 +3,8 @@ import cbt._ class Build(val context: cbt.Context) extends BaseBuild{ override def dependencies = super.dependencies :+ context.cbtDependency def apply = run + override def run = { + classes.flatMap( lib.discoverCbtMain ).head( context ) + } def args = context.args } diff --git a/test/test.scala b/test/test.scala index dbf4f9e..58c5246 100644 --- a/test/test.scala +++ b/test/test.scala @@ -8,12 +8,13 @@ import scala.concurrent._ import scala.concurrent.duration._ // micro framework object Main{ - def main(_args: Array[String]): Unit = { - val start = System.currentTimeMillis + def cbtMain(context: Context): ExitCode = { + import context._ + val _args = context.args val args = new Stage1ArgsParser(_args.toVector) implicit val logger: Logger = new Logger(args.enabledLoggers, System.currentTimeMillis) val lib = new Lib(logger) - val cbtHome = new File(System.getenv("CBT_HOME")) + val mavenCache = cache ++ "/maven" val slow = ( System.getenv("CIRCLECI") != null // enable only on circle @@ -21,9 +22,13 @@ object Main{ ) val compat = !args.args.contains("no-compat") val shellcheck = !args.args.contains("no-shellcheck") + val fork = args.args.contains("fork") + val direct = args.args.contains("direct") if(!slow) System.err.println( "Skipping slow tests" ) if(!compat) System.err.println( "Skipping cbt version compatibility tests" ) + if(fork) System.err.println( "Forking tests" ) + if(direct) System.err.println( "Running tests in direct mode" ) if(shellcheck){ val pb = new ProcessBuilder( "/usr/bin/env", "shellcheck", (cbtHome / "cbt").string ) @@ -59,22 +64,66 @@ object Main{ def runCbt(path: String, _args: Seq[String])(implicit logger: Logger): Result = { import java.io._ - val allArgs: Seq[String] = ((cbtHome.string ++ "/cbt") +: "direct" +: (_args ++ args.propsRaw)) - logger.test(allArgs.toString) - val pb = new ProcessBuilder( allArgs :_* ) - pb.directory(cbtHome ++ ("/test/" ++ path)) - val p = pb.start - val serr = new InputStreamReader(p.getErrorStream); - val sout = new InputStreamReader(p.getInputStream); - import scala.concurrent.ExecutionContext.Implicits.global - val err = Future(blocking(Iterator.continually(serr.read).takeWhile(_ != -1).map(_.toChar).mkString)) - val out = Future(blocking(Iterator.continually(sout.read).takeWhile(_ != -1).map(_.toChar).mkString)) - p.waitFor - Result( - p.exitValue == 0, - Await.result( out, Duration.Inf ), - Await.result( err, Duration.Inf ) - ) + val workingDirectory = cbtHome / "test" / path + if( fork ){ + val allArgs = Seq((cbtHome / "cbt").string) ++ (if(direct) Seq("direct") else Nil) ++ _args ++ args.propsRaw + logger.test(allArgs.toString) + val pb = new ProcessBuilder( allArgs :_* ) + pb.directory( workingDirectory ) + val p = pb.start + val serr = new InputStreamReader(p.getErrorStream); + val sout = new InputStreamReader(p.getInputStream); + import scala.concurrent.ExecutionContext.Implicits.global + val err = Future(blocking(Iterator.continually(serr.read).takeWhile(_ != -1).map(_.toChar).mkString)) + val out = Future(blocking(Iterator.continually(sout.read).takeWhile(_ != -1).map(_.toChar).mkString)) + p.waitFor + p.exitValue + Result( + p.exitValue === 0, + Await.result( out, Duration.Inf ), + Await.result( err, Duration.Inf ) + ) + } else { + val c = context.copy( + workingDirectory = workingDirectory, + args = _args.drop(1), + transientCache = new java.util.HashMap() + ) + val ( outVar, errVar ) = lib.getOutErr + val oldOut = outVar.get + val oldErr = errVar.get + val out = new ByteArrayOutputStream + val err = new ByteArrayOutputStream + val out2 = new ByteArrayOutputStream + val err2 = new ByteArrayOutputStream + try{ + outVar.set(new PrintStream(out)) + errVar.set(new PrintStream(err)) + val exitValue = try{ + scala.Console.withOut(out2)( + scala.Console.withErr(err2)( + lib.trapExitCode( + lib.callReflective( DirectoryDependency(c,None), _args.headOption, c ) + ) + ) + ) + } catch { + case scala.util.control.NonFatal(e) => + lib.redirectOutToErr( e.printStackTrace ) + ExitCode.Failure + } + System.out.flush + System.err.flush + Result( + exitValue.integer === 0, + out.toString ++ out2.toString, + err.toString ++ err2.toString + ) + } finally { + outVar.set(oldOut) + errVar.set(oldErr) + } + } } case class Result(exit0: Boolean, out: String, err: String) def assertSuccess(res: Result, msg: => String)(implicit logger: Logger) = { @@ -122,33 +171,12 @@ object Main{ logger.test( "Running tests " ++ _args.toList.toString ) - val cache = cbtHome ++ "/cache" - val mavenCache = cache ++ "/maven" - val cbtLastModified = System.currentTimeMillis implicit val transientCache: java.util.Map[AnyRef,AnyRef] = new java.util.HashMap implicit val classLoaderCache: ClassLoaderCache = new ClassLoaderCache( new java.util.HashMap ) def Resolver(urls: URL*) = MavenResolver(cbtLastModified, mavenCache, urls: _*) { - val noContext = new ContextImplementation( - cbtHome ++ "/test/nothing", - cbtHome, - Array(), - Array(), - start, - cbtLastModified, - null, - new HashMap[AnyRef,AnyRef], - new HashMap[AnyRef,AnyRef], - cache, - cbtHome, - cbtHome, - cbtHome ++ "/compatibilityTarget", - null, - false - ) - - val b = new BasicBuild(noContext){ + val b = new BasicBuild(context){ override def dependencies = Resolver(mavenCentral).bind( MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0"), @@ -250,7 +278,7 @@ object Main{ } compile("../examples/uber-jar-example") - if( compat ){ + if( compat && fork ){ // FIXME: this should not be excluded in forking val res = task("docJar","simple-fixed-cbt") assert( res.out endsWith "simple-fixed-cbt_2.11-0.1-javadoc.jar\n", res.out ) assert( res.err contains "model contains", res.err ) @@ -411,6 +439,6 @@ object Main{ System.err.println(" DONE!") System.err.println( successes.toString ++ " succeeded, "++ failures.toString ++ " failed" ) - if(failures > 0) System.exit(1) else System.exit(0) + if(failures > 0) ExitCode.Failure else ExitCode.Success } } -- cgit v1.2.3 From 9049773eab2dd7675a9b1cacd92bbe67825cf1ea Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sun, 19 Mar 2017 19:42:19 -0400 Subject: explicitly convert to Vector everywhere for hopefully performance benefits --- coursier/Coursier.scala | 2 +- plugins/scalatest/ScalaTest.scala | 2 +- stage1/MavenRepository.scala | 4 ++-- stage1/Stage1Lib.scala | 2 +- stage1/cbt.scala | 10 +++++----- stage1/resolver.scala | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/coursier/Coursier.scala b/coursier/Coursier.scala index 8a66aee..f5727a7 100644 --- a/coursier/Coursier.scala +++ b/coursier/Coursier.scala @@ -40,7 +40,7 @@ object Coursier{ case Right(file) => file }) - resolution.dependencies.map( d => cbt.JavaDependency(d.module.organization,d.module.name, d.version)).to[collection.immutable.Seq] + resolution.dependencies.map( d => cbt.JavaDependency(d.module.organization,d.module.name, d.version)).toVector } } } diff --git a/plugins/scalatest/ScalaTest.scala b/plugins/scalatest/ScalaTest.scala index 7f805fd..e2f44e3 100644 --- a/plugins/scalatest/ScalaTest.scala +++ b/plugins/scalatest/ScalaTest.scala @@ -33,7 +33,7 @@ object ScalaTestLib{ .getMethod("discoverSuiteNames", classOf[List[_]], classOf[ClassLoader], classOf[Option[_]]) .invoke(null, List(discoveryPath.string ++ "/"), classLoader, None) .asInstanceOf[Set[String]] - .to + .toVector } def loadSuite(name: String, classLoader: ClassLoader) = { classLoader.loadClass(name).getConstructor().newInstance().asInstanceOf[Suite] diff --git a/stage1/MavenRepository.scala b/stage1/MavenRepository.scala index 6be537b..9ada1fd 100644 --- a/stage1/MavenRepository.scala +++ b/stage1/MavenRepository.scala @@ -7,7 +7,7 @@ case class MavenResolver( implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef], classLoaderCache: ClassLoaderCache ){ def bind( dependencies: MavenDependency* ): Seq[BoundMavenDependency] - = dependencies.map( BoundMavenDependency(cbtLastModified,mavenCache,_,urls.to) ).to + = dependencies.map( BoundMavenDependency(cbtLastModified,mavenCache,_,urls.toVector) ).toVector def bindOne( dependency: MavenDependency ): BoundMavenDependency - = BoundMavenDependency( cbtLastModified, mavenCache, dependency, urls.to ) + = BoundMavenDependency( cbtLastModified, mavenCache, dependency, urls.toVector ) } diff --git a/stage1/Stage1Lib.scala b/stage1/Stage1Lib.scala index 8aaa6e6..9e6f6b0 100644 --- a/stage1/Stage1Lib.scala +++ b/stage1/Stage1Lib.scala @@ -375,7 +375,7 @@ ${sourceFiles.sorted.mkString(" \\\n")} def classLoaderRecursion( dependency: Dependency, latest: Map[(String,String),Dependency])(implicit transientCache: java.util.Map[AnyRef,AnyRef], cache: ClassLoaderCache): ClassLoader = { // FIXME: shouldn't we be using KeyLockedLazyCache instead of hashmap directly here? - val dependencies = dependency.dependencies + val dependencies = dependency.dependencies.toVector val dependencyClassLoader: ClassLoader = { if( dependency.dependencies.isEmpty ){ NailgunLauncher.jdkClassLoader diff --git a/stage1/cbt.scala b/stage1/cbt.scala index 64257c2..52cebe7 100644 --- a/stage1/cbt.scala +++ b/stage1/cbt.scala @@ -61,10 +61,10 @@ object `package`{ implicit class DependencyExtensions(subject: Dependency){ import subject._ def dependencyClasspath(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef], classLoaderCache: ClassLoaderCache): ClassPath - = Dependencies(dependenciesArray.to).classpath - def exportedClasspath: ClassPath = ClassPath(exportedClasspathArray.to) + = Dependencies(dependenciesArray.toVector).classpath + def exportedClasspath: ClassPath = ClassPath(exportedClasspathArray.toVector) def classpath(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef], classLoaderCache: ClassLoaderCache) = exportedClasspath ++ dependencyClasspath - def dependencies: Seq[Dependency] = dependenciesArray.to + def dependencies: Seq[Dependency] = dependenciesArray.toVector } implicit class ContextExtensions(subject: Context){ import subject._ @@ -78,8 +78,8 @@ object `package`{ } val cbtDependency = cbtDependencies.stage2Dependency - def args: Seq[String] = argsArray.to - def enabledLoggers: Set[String] = enabledLoggersArray.to + def args: Seq[String] = argsArray.toVector + def enabledLoggers: Set[String] = enabledLoggersArray.toSet def scalaVersion = Option(scalaVersionOrNull) def parentBuild = Option(parentBuildOrNull) def cbtLastModified: scala.Long = subject.cbtLastModified diff --git a/stage1/resolver.scala b/stage1/resolver.scala index 0f0acaa..57ac5b6 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -365,7 +365,7 @@ case class BoundMavenDependency( } ).map( BoundMavenDependency( cbtLastModified, mavenCache, _, repositories, replace ) - ).to + ).toVector } def lookup( xml: Node, accessor: Node => NodeSeq ): Option[String] = { // println("lookup in " + xml) -- cgit v1.2.3 From 88854cfeb423a414296ffe2b04938d1b99fb4868 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sun, 19 Mar 2017 19:54:29 -0400 Subject: io performance tweak: cache cbtLastModified --- stage1/Stage1.scala | 11 +++++++++-- stage1/cbt.scala | 2 +- stage1/resolver.scala | 3 +-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/stage1/Stage1.scala b/stage1/Stage1.scala index c243cc0..d9bde7c 100644 --- a/stage1/Stage1.scala +++ b/stage1/Stage1.scala @@ -98,7 +98,8 @@ object Stage1{ val cls = this.getClass.getClassLoader.loadClass("cbt.NailgunLauncher") - def cbtDependencies = new CbtDependencies( + def _cbtDependencies = new CbtDependencies( + (stage2Target++".last-success").lastModified, mavenCache, nailgunTarget, stage1Target, stage2Target, new File(buildStage1.compatibilityClasspath) ) @@ -109,12 +110,18 @@ object Stage1{ val Some( stage2LastModified ) = compile( buildStage1.stage1LastModified, stage2sourceFiles, stage2Target, stage2StatusFile, - cbtDependencies.stage2Dependency.dependencies, + _cbtDependencies.stage2Dependency.dependencies, mavenCache, Seq("-deprecation","-feature","-unchecked"), zincVersion = constants.zincVersion, scalaVersion = constants.scalaVersion ) + def cbtDependencies = new CbtDependencies( + (stage2Target++".last-success").lastModified, + mavenCache, nailgunTarget, stage1Target, stage2Target, + new File(buildStage1.compatibilityClasspath) + ) + logger.stage1(s"calling CbtDependency.classLoader") assert( diff --git a/stage1/cbt.scala b/stage1/cbt.scala index 52cebe7..039f779 100644 --- a/stage1/cbt.scala +++ b/stage1/cbt.scala @@ -74,7 +74,7 @@ object `package`{ def classLoaderCache: ClassLoaderCache = new ClassLoaderCache( persistentCache ) def cbtDependencies = { import paths._ - new CbtDependencies(mavenCache, nailgunTarget, stage1Target, stage2Target, compatibilityTarget)(logger, transientCache, classLoaderCache) + new CbtDependencies(cbtLastModified, mavenCache, nailgunTarget, stage1Target, stage2Target, compatibilityTarget)(logger, transientCache, classLoaderCache) } val cbtDependency = cbtDependencies.stage2Dependency diff --git a/stage1/resolver.scala b/stage1/resolver.scala index 57ac5b6..1eaeebd 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -176,9 +176,8 @@ case class PostBuildDependency(target: File, _dependencies: Seq[DependencyImplem override def exportedClasspath = ClassPath( Seq(target) ) override def dependencies = _dependencies } -case class CbtDependencies(mavenCache: File, nailgunTarget: File, stage1Target: File, stage2Target: File, compatibilityTarget: File)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef], classLoaderCache: ClassLoaderCache){ +case class CbtDependencies(cbtLastModified: Long, mavenCache: File, nailgunTarget: File, stage1Target: File, stage2Target: File, compatibilityTarget: File)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef], classLoaderCache: ClassLoaderCache){ val compatibilityDependency = PostBuildDependency(compatibilityTarget, Nil) - val cbtLastModified = (stage2Target++".last-success").lastModified val stage1Dependency = PostBuildDependency( stage1Target, Seq( -- cgit v1.2.3 From 0a6f67c70d5f878d3045bbac9b70f94859eb47d9 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sun, 19 Mar 2017 19:57:20 -0400 Subject: test for MavenDependency equality --- test/test.scala | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/test.scala b/test/test.scala index 58c5246..5c107fe 100644 --- a/test/test.scala +++ b/test/test.scala @@ -187,6 +187,13 @@ object Main{ assert(cp.strings.distinct == cp.strings, "duplicates in classpath: " ++ cp.string) } + { + def d = Resolver(mavenCentral).bindOne( + MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0") + ) + assert(d === d) + } + // test that messed up artifacts crash with an assertion (which should tell the user what's up) assertException[AssertionError](){ Resolver(mavenCentral).bindOne( MavenDependency("com.jcraft", "jsch", " 0.1.53") ).classpath -- cgit v1.2.3 From 52adb81450955472fe4ecd011615b80f6289def6 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sun, 19 Mar 2017 19:58:36 -0400 Subject: cache friendly equality for builds --- stage2/BasicBuild.scala | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/stage2/BasicBuild.scala b/stage2/BasicBuild.scala index ebb6a40..18b1754 100644 --- a/stage2/BasicBuild.scala +++ b/stage2/BasicBuild.scala @@ -6,6 +6,13 @@ import java.nio.file._ class BasicBuild(final val context: Context) extends BaseBuild trait BaseBuild extends BuildInterface with DependencyImplementation with SbtDependencyDsl{ + override def equals(other: Any) = { + other match { + case b: BaseBuild => projectDirectory === b.projectDirectory + case _ => false + } + } + //* DO NOT OVERRIDE CONTEXT in non-idempotent ways, because .copy and new Build // will create new instances given the context, which means operations in the // overrides will happen multiple times and if they are not idempotent stuff likely breaks -- cgit v1.2.3 From 6b37681f94056f05cf545fbb1e483dabbfa8492c Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sun, 19 Mar 2017 20:01:09 -0400 Subject: performance tweak: cache dependencyClasspath (major) --- stage1/resolver.scala | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/stage1/resolver.scala b/stage1/resolver.scala index 1eaeebd..e485823 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -119,11 +119,13 @@ trait DependencyImplementation extends Dependency{ // FIXME: these probably need to update outdated as well def classpath : ClassPath = exportedClasspath ++ dependencyClasspath - def dependencyClasspath : ClassPath = ClassPath( - transitiveDependencies - .flatMap(_.exportedClasspath.files) - .distinct // <- currently needed here to handle diamond dependencies on builds (duplicate in classpath) - ) + def dependencyClasspath : ClassPath = taskCache[DependencyImplementation]( "dependencyClasspath" ).memoize{ + ClassPath( + transitiveDependencies + .flatMap(_.exportedClasspath.files) + .distinct // <- currently needed here to handle diamond dependencies on builds (duplicate in classpath) + ) + } def dependencies: Seq[Dependency] /** return dependencies in order of linearized dependence. this is a bit tricky. */ -- cgit v1.2.3 From d24aa27365015d024d5b11dbbfbbe5f3fceda4fb Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sun, 19 Mar 2017 20:01:34 -0400 Subject: performance tweak: cache classloader --- stage1/resolver.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage1/resolver.scala b/stage1/resolver.scala index e485823..6c31a98 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -96,7 +96,7 @@ trait DependencyImplementation extends Dependency{ ) def mainMethods = classes.flatMap( lib.discoverMain ) - def classLoader: ClassLoader = { + def classLoader: ClassLoader = taskCache[DependencyImplementation]( "classLoader" ).memoize{ if( flatClassLoader ){ new java.net.URLClassLoader(classpath.strings.map(f => new URL("file://" ++ f)).toArray) } else { -- cgit v1.2.3 From 4185639097f8dc546c6474095c1fe853d43a1069 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sun, 19 Mar 2017 20:03:08 -0400 Subject: performance tweak: cache results of parsed pom files in memory --- stage1/Stage1Lib.scala | 46 +++++++++++++++++++++++++++++++--------------- stage1/resolver.scala | 13 ++++++++++--- 2 files changed, 41 insertions(+), 18 deletions(-) diff --git a/stage1/Stage1Lib.scala b/stage1/Stage1Lib.scala index 9e6f6b0..3ffc878 100644 --- a/stage1/Stage1Lib.scala +++ b/stage1/Stage1Lib.scala @@ -310,23 +310,39 @@ ${sourceFiles.sorted.mkString(" \\\n")} groupId, artifactId ++ "_" ++ scalaMajorVersion, version, classifier, verifyHash ) - def cacheOnDisk[T] - ( cbtLastModified: Long, cacheFile: File ) + def cacheOnDisk[T,J <: AnyRef : scala.reflect.ClassTag] + ( cbtLastModified: Long, cacheFile: File, persistentCache: java.util.Map[AnyRef,AnyRef] ) ( deserialize: String => T ) ( serialize: T => String ) - ( compute: => Seq[T] ) = { - if(cacheFile.exists && cacheFile.lastModified > cbtLastModified ){ - import collection.JavaConversions._ - Files - .readAllLines( cacheFile.toPath, StandardCharsets.UTF_8 ) - .toStream - .map(deserialize) - } else { - val result = compute - val string = result.map(serialize).mkString("\n") - write(cacheFile, string) - result - } + ( dejavafy: J => Seq[T] ) + ( javafy: Seq[T] => J ) + ( compute: => Seq[T] ): Seq[T] = { + val key = "cacheOnDisk:" + cacheFile + Option( persistentCache.get(key) ).map( + _.asInstanceOf[Array[AnyRef]] + ).map{ + case Array(time: java.lang.Long, javafied: J) => (time, javafied) + }.filter( _._1 > cbtLastModified ) + .map( _._2 ) + .map( dejavafy ) + .orElse{ + (cacheFile.exists && cacheFile.lastModified > cbtLastModified).option{ + import collection.JavaConversions._ + val v = Files + .readAllLines( cacheFile.toPath, StandardCharsets.UTF_8 ) + .toStream + .map( deserialize ) + persistentCache.put(key, Array(System.currentTimeMillis:java.lang.Long, javafy(v))) + v + } + }.getOrElse{ + val result = compute + val strings = result.map(serialize) + val string = strings.mkString("\n") + write(cacheFile, string) + persistentCache.put(key, Array(System.currentTimeMillis:java.lang.Long, javafy(result))) + result + } } def dependencyTreeRecursion(root: Dependency, indent: Int = 0): String = ( diff --git a/stage1/resolver.scala b/stage1/resolver.scala index 6c31a98..1293a89 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -213,12 +213,19 @@ abstract class DependenciesProxy{ case class MavenDependency( groupId: String, artifactId: String, version: String, classifier: Classifier = Classifier.none, verifyHash: Boolean = true ){ - private[cbt] def serialize = groupId ++ ":" ++ artifactId ++ ":"++ version ++ classifier.name.map(":" ++ _).getOrElse("") + private[cbt] def serialize = // PERFORMANCE HOTSPOT + groupId + ":" + artifactId + ":" + version + ( if(classifier.name.nonEmpty) ":" + classifier.name.get else "" ) + private[cbt] def javafy: Array[String] = + Array(groupId,artifactId,version) ++ classifier.name } object MavenDependency{ private[cbt] def deserialize = (_:String).split(":") match { case col => MavenDependency( col(0), col(1), col(2), Classifier(col.lift(3)) ) } + private[cbt] def dejavafy = + ( cols:Array[Array[String]] ) => cols.map( + col => MavenDependency( col(0), col(1), col(2), Classifier(col.lift(3)) ) + ).toSeq } // FIXME: take MavenResolver instead of mavenCache and repositories separately case class BoundMavenDependency( @@ -338,8 +345,8 @@ case class BoundMavenDependency( if(classifier == Classifier.sources) Seq() else { lib.cacheOnDisk( - cbtLastModified, mavenCache ++ basePath(true) ++ ".pom.dependencies" - )( MavenDependency.deserialize )( _.serialize ){ + cbtLastModified, mavenCache ++ basePath(true) ++ ".pom.dependencies", classLoaderCache.hashMap + )( MavenDependency.deserialize )( _.serialize )( MavenDependency.dejavafy )( _.map(_.javafy).toArray ){ (pomXml \ "dependencies" \ "dependency").collect{ case xml if ( (xml \ "scope").text == "" || (xml \ "scope").text == "compile" ) && (xml \ "optional").text != "true" => val artifactId = lookup(xml,_ \ "artifactId").get -- cgit v1.2.3 From 90df89c5de90e69e7fdada655b169d0e48de0f57 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sun, 19 Mar 2017 20:03:28 -0400 Subject: performance tweak: cache lastModified --- stage1/resolver.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stage1/resolver.scala b/stage1/resolver.scala index 1293a89..4f6a2ef 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -259,7 +259,9 @@ case class BoundMavenDependency( ) override def show: String = this.getClass.getSimpleName ++ "(" ++ mavenDependency.serialize ++ ")" - override final lazy val lastModified = classpath.strings.map(new File(_).lastModified).max + override final lazy val lastModified: Long = taskCache[BoundMavenDependency]( "lastModified" ).memoize[java.lang.Long]{ + classpath.strings.map(new File(_).lastModified).max + } private val groupPath = groupId.split("\\.").mkString("/") protected[cbt] def basePath(useClassifier: Boolean) = s"/$groupPath/$artifactId/$version/$artifactId-$version" ++ (if (useClassifier) classifier.name.map("-"++_).getOrElse("") else "") -- cgit v1.2.3 From a29fa8b51a167102d807330ce0c65d015e5c44e3 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sun, 19 Mar 2017 20:04:08 -0400 Subject: performance tweak: cache hashes in memory --- stage1/resolver.scala | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/stage1/resolver.scala b/stage1/resolver.scala index 4f6a2ef..866105e 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -286,10 +286,15 @@ case class BoundMavenDependency( } private def resolveHash(suffix: String, useClassifier: Boolean) = { - Files.readAllLines( - resolve( suffix ++ ".sha1", None, useClassifier ).toPath, - StandardCharsets.UTF_8 - ).mkString("\n").split(" ").head.trim + val path = resolve( suffix ++ ".sha1", None, useClassifier ).toPath + Option( classLoaderCache.hashMap.get("hash:"+path) ).map(_.asInstanceOf[String]).getOrElse{ + val result = Files.readAllLines( + path, + StandardCharsets.UTF_8 + ).mkString("\n").split(" ").head.trim + classLoaderCache.hashMap.put("hash:"+path, result) + result + } } def jarSha1: String = taskCache[BoundMavenDependency]("jarSha1").memoize{ resolveHash("jar", true) } -- cgit v1.2.3 From e5940451943d360193af8fb52e65c3eadd980f87 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sun, 19 Mar 2017 20:05:51 -0400 Subject: performance tweak: cache maven URI path --- stage1/resolver.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/stage1/resolver.scala b/stage1/resolver.scala index 866105e..7866fcb 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -263,8 +263,9 @@ case class BoundMavenDependency( classpath.strings.map(new File(_).lastModified).max } - private val groupPath = groupId.split("\\.").mkString("/") - protected[cbt] def basePath(useClassifier: Boolean) = s"/$groupPath/$artifactId/$version/$artifactId-$version" ++ (if (useClassifier) classifier.name.map("-"++_).getOrElse("") else "") + private lazy val base = "/" + groupId.split("\\.").mkString("/") + "/" + artifactId + "/" + version + "/" + artifactId + "-" + version + protected[cbt] def basePath(useClassifier: Boolean) = // PERFORMANCE HOTSPOT + base + (if (useClassifier && classifier.name.nonEmpty) "-" + classifier.name.get else "") //private def coursierJarFile = userHome++"/.coursier/cache/v1/https/repo1.maven.org/maven2"++basePath++".jar" -- cgit v1.2.3 From f5fe60ada0a550907a378592ee07291ee157f275 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sun, 19 Mar 2017 20:52:29 -0400 Subject: performance: cache moduleKey and make it’s string concat quicker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- stage1/resolver.scala | 8 ++++---- stage2/BasicBuild.scala | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/stage1/resolver.scala b/stage1/resolver.scala index 7866fcb..5090af7 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -158,13 +158,13 @@ case class BinaryDependency( paths: Seq[File], dependencies: Seq[Dependency] )(i def exportedClasspath = ClassPath(paths) override def lastModified = paths.map(_.lastModifiedRecursive).max // FIXME: cache this def targetClasspath = exportedClasspath - def moduleKey = this.getClass.getName ++ "(" ++ paths.mkString(", ") ++ ")" + lazy val moduleKey = this.getClass.getName + "(" + paths.mkString(", ") + ")" // PERFORMANCE HOTSPOT } /** Allows to easily assemble a bunch of dependencies */ case class Dependencies( dependencies: Seq[Dependency] )(implicit val logger: Logger, val transientCache: java.util.Map[AnyRef,AnyRef], val classLoaderCache: ClassLoaderCache) extends DependencyImplementation{ override def lastModified = dependencies.map(_.lastModified).maxOption.getOrElse(0) - def moduleKey = this.getClass.getName ++ "(" ++ dependencies.map(_.moduleKey).mkString(", ") ++ ")" + lazy val moduleKey = this.getClass.getName + "(" + dependencies.map(_.moduleKey).mkString(", ") + ")" // PERFORMANCE HOTSPOT def targetClasspath = ClassPath() def exportedClasspath = ClassPath() override def show: String = this.getClass.getSimpleName + "( " + dependencies.map(_.show).mkString(", ") + " )" @@ -172,7 +172,7 @@ case class Dependencies( dependencies: Seq[Dependency] )(implicit val logger: Lo case class PostBuildDependency(target: File, _dependencies: Seq[DependencyImplementation])(implicit val logger: Logger, val transientCache: java.util.Map[AnyRef,AnyRef], val classLoaderCache: ClassLoaderCache) extends DependencyImplementation{ override final lazy val lastModified = (target++".last-success").lastModified - def moduleKey = target.string + lazy val moduleKey = target.string override def show = s"PostBuildDependency($target)" override def targetClasspath = exportedClasspath override def exportedClasspath = ClassPath( Seq(target) ) @@ -234,7 +234,7 @@ case class BoundMavenDependency( )( implicit val logger: Logger, val transientCache: java.util.Map[AnyRef,AnyRef], val classLoaderCache: ClassLoaderCache ) extends ArtifactInfo with DependencyImplementation{ - def moduleKey = this.getClass.getName ++ "(" ++ mavenDependency.serialize ++ ")" + lazy val moduleKey = this.getClass.getName + "(" + mavenDependency.serialize + ")" // PERFORMANCE HOTSPOT override def hashCode = mavenDependency.hashCode override def equals(other: Any) = other match{ case o: BoundMavenDependency => o.mavenDependency == mavenDependency && o.repositories == repositories diff --git a/stage2/BasicBuild.scala b/stage2/BasicBuild.scala index 18b1754..067d1b7 100644 --- a/stage2/BasicBuild.scala +++ b/stage2/BasicBuild.scala @@ -17,7 +17,7 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with SbtDep // will create new instances given the context, which means operations in the // overrides will happen multiple times and if they are not idempotent stuff likely breaks def context: Context - def moduleKey: String = "BaseBuild("+target.string+")" + lazy val moduleKey: String = "BaseBuild("+target.string+")" implicit def transientCache: java.util.Map[AnyRef,AnyRef] = context.transientCache implicit def libraries(implicit context: Context): libraries = new libraries(context) -- cgit v1.2.3 From b0385cedf55c3e04171f12ce2775010a269800a5 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sun, 19 Mar 2017 20:54:48 -0400 Subject: performance: avoid compiling regex every time --- stage1/resolver.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage1/resolver.scala b/stage1/resolver.scala index 5090af7..0e5d221 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -416,7 +416,7 @@ case class BoundMavenDependency( } } object BoundMavenDependency{ - def ValidIdentifier = "^([A-Za-z0-9_\\-.]+)$".r // according to maven's DefaultModelValidator.java + val ValidIdentifier = "^([A-Za-z0-9_\\-.]+)$".r // according to maven's DefaultModelValidator.java def semanticVersionLessThan(left: Array[Either[Int,String]], right: Array[Either[Int,String]]) = { // FIXME: this ignores ends when different size val zipped = left zip right -- cgit v1.2.3 From ac92c2af65a064da2a6ac94d30071ba6aa8ac04d Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sun, 19 Mar 2017 20:54:21 -0400 Subject: performance: avoid io call when building classpath string --- stage1/ClassPath.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/stage1/ClassPath.scala b/stage1/ClassPath.scala index d8568c2..539efc7 100644 --- a/stage1/ClassPath.scala +++ b/stage1/ClassPath.scala @@ -22,6 +22,9 @@ case class ClassPath(files: Seq[File] = Seq()){ def ++(other: ClassPath) = ClassPath(files ++ other.files) def string = strings.mkString( File.pathSeparator ) def strings = files.map{ - f => f.string ++ ( if(f.isDirectory) "/" else "" ) + f => f.string + ( + // using file extension instead of isDirectory for performance reasons + if( f.getName.endsWith(".jar") /* !f.isDirectory */ ) "" else "/" + ) }.sorted } -- cgit v1.2.3