From 00d9485f5597fdecc58461bd81df635fafbe494f Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Fri, 25 Nov 2016 16:48:28 -0500 Subject: Merge separate hashmaps for persistent cache into one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This isn’t type-safe, but re-using that same hashmap for both keys and classloaders allows to reduce the number of members in Context. Also we can re-use the same hashMap for other things as well in the coming commits, e.g. timestamps. --- stage1/ContextImplementation.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'stage1/ContextImplementation.scala') diff --git a/stage1/ContextImplementation.scala b/stage1/ContextImplementation.scala index 152e606..6eb2e53 100644 --- a/stage1/ContextImplementation.scala +++ b/stage1/ContextImplementation.scala @@ -11,9 +11,8 @@ case class ContextImplementation( startCompat: Long, cbtHasChangedCompat: Boolean, scalaVersionOrNull: String, - permanentKeys: ConcurrentHashMap[String,AnyRef], - permanentClassLoaders: ConcurrentHashMap[AnyRef,ClassLoader], - taskCache: ConcurrentHashMap[AnyRef,AnyRef], + persistentCache: ConcurrentHashMap[AnyRef,AnyRef], + transientCache: ConcurrentHashMap[AnyRef,AnyRef], cache: File, cbtHome: File, cbtRootHome: File, -- cgit v1.2.3 From e56f8afa03035140280bc8d3d878ad225def381e Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sun, 27 Nov 2016 16:00:58 -0500 Subject: replace flawed concurrent hashmap cache with consistent replacement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The concurrent hashmap approach to classloader caching was flawed. Assume you have two concurrently running builds A and B and projects P2 and P3 depending on project P1. And assume a time sequence where A compiles P1, then compiles P2, then P1’s sources change, then B compiles P1, then A compiles P3. At the end P2 and P3 will have different versions of P1 as their parent classloaders. This is inconsistent. The easiest way to work around this is making sure only one thread is changing the classloader cache during it’s entire run. This would mean either no concurrency or what we have done here, which is letting threads work on a copy of the cache and replace the original cache in the end using an atomic operation. This means the thread that finishes last wins, but for caching that’s fine. Worst case some things aren’t cached in a concurrent execution. This change also means that we don’t need concurrent hashmaps for the classloader cache anymore since no two theads will access the same hashmap. We still need a concurrent hashmap for the class caches inside of the classloaders as multiple threads can access the same classloaders. --- compatibility/Context.java | 6 ++--- nailgun_launcher/CbtURLClassLoader.java | 26 ++++++++++---------- nailgun_launcher/JavaCache.java | 5 ++-- nailgun_launcher/NailgunLauncher.java | 42 +++++++++++++++++---------------- stage1/ClassLoaderCache.scala | 4 ++-- stage1/ContextImplementation.scala | 5 ++-- stage1/KeyLockedLazyCache.scala | 4 +--- stage1/PoorMansProfiler.scala | 4 ++-- stage1/Stage1.scala | 4 ++-- stage1/Stage1Lib.scala | 1 - stage1/cbt.scala | 1 - stage1/resolver.scala | 2 -- stage2/Stage2.scala | 5 ++-- test/test.scala | 6 ++--- 14 files changed, 56 insertions(+), 59 deletions(-) (limited to 'stage1/ContextImplementation.scala') diff --git a/compatibility/Context.java b/compatibility/Context.java index 5a0f9c6..a7af740 100644 --- a/compatibility/Context.java +++ b/compatibility/Context.java @@ -1,6 +1,6 @@ package cbt; import java.io.*; -import java.util.concurrent.ConcurrentHashMap; +import java.util.*; // TODO: try to reduce the number of members public abstract class Context{ @@ -11,8 +11,8 @@ public abstract class Context{ public abstract Long startCompat(); public abstract Boolean cbtHasChangedCompat(); public abstract String scalaVersionOrNull(); // needed to propagate scalaVersion to dependendee builds - public abstract ConcurrentHashMap persistentCache(); - public abstract ConcurrentHashMap transientCache(); + public abstract Map persistentCache(); + public abstract Map transientCache(); public abstract File cache(); public abstract File cbtHome(); public abstract File cbtRootHome(); // REMOVE diff --git a/nailgun_launcher/CbtURLClassLoader.java b/nailgun_launcher/CbtURLClassLoader.java index e3d597e..43d07f4 100644 --- a/nailgun_launcher/CbtURLClassLoader.java +++ b/nailgun_launcher/CbtURLClassLoader.java @@ -23,19 +23,21 @@ public class CbtURLClassLoader extends java.net.URLClassLoader{ } public Class loadClass(String name, Boolean resolve) throws ClassNotFoundException{ //System.out.println("loadClass("+name+") on \n"+this); - if(!cache.contains(name)) - try{ - cache.put(super.loadClass(name, resolve), name); - } catch (ClassNotFoundException e){ - cache.put(Object.class, name); + synchronized( cache ){ + if(!cache.contains(name)) + try{ + cache.put(super.loadClass(name, resolve), name); + } catch (ClassNotFoundException e){ + cache.put(Object.class, name); + } + Class _class = cache.get(name); + if(_class == Object.class){ + if( name == "java.lang.Object" ) + return Object.class; + else return null; + } else { + return _class; } - Class _class = cache.get(name); - if(_class == Object.class){ - if( name == "java.lang.Object" ) - return Object.class; - else return null; - } else { - return _class; } } void assertExist(URL[] urls){ diff --git a/nailgun_launcher/JavaCache.java b/nailgun_launcher/JavaCache.java index 56730df..3ba12ab 100644 --- a/nailgun_launcher/JavaCache.java +++ b/nailgun_launcher/JavaCache.java @@ -1,15 +1,14 @@ package cbt; import java.util.*; -import java.util.concurrent.ConcurrentHashMap; import static java.io.File.pathSeparator; import static cbt.Stage0Lib.*; final class JavaCache{ - ConcurrentHashMap hashMap; + Map hashMap; public JavaCache( - ConcurrentHashMap hashMap + Map hashMap ){ this.hashMap = hashMap; } diff --git a/nailgun_launcher/NailgunLauncher.java b/nailgun_launcher/NailgunLauncher.java index 0b41888..c397810 100644 --- a/nailgun_launcher/NailgunLauncher.java +++ b/nailgun_launcher/NailgunLauncher.java @@ -4,7 +4,6 @@ import java.lang.reflect.*; import java.net.*; import java.security.*; import java.util.*; -import java.util.concurrent.ConcurrentHashMap; import static cbt.Stage0Lib.*; import static java.io.File.pathSeparator; @@ -15,9 +14,7 @@ import static java.io.File.pathSeparator; */ public class NailgunLauncher{ /** Persistent cache for caching classloaders for the JVM life time. */ - private final static JavaCache classLoaderCache = new JavaCache( - new ConcurrentHashMap() - ); + private static Map classLoaderCacheHashMap = new HashMap(); public final static SecurityManager initialSecurityManager = System.getSecurityManager(); @@ -35,7 +32,7 @@ public class NailgunLauncher{ ((File) get(context, "cbtHome")).toString(), ((File) get(context, "compatibilityTarget")).toString() + "/", new JavaCache( - (ConcurrentHashMap) get(context, "persistentCache") + (HashMap) get(context, "persistentCache") ) ); return @@ -79,28 +76,33 @@ public class NailgunLauncher{ String CBT_HOME = System.getenv("CBT_HOME"); String cache = CBT_HOME + "/cache/"; String compatibilityTarget = CBT_HOME + "/compatibility/" + TARGET; + // copy cache, so that this thread has a consistent view + // replace before returning, see below + JavaCache classLoaderCache = new JavaCache( + new HashMap(classLoaderCacheHashMap) + ); BuildStage1Result res = buildStage1( false, start, cache, CBT_HOME, compatibilityTarget, classLoaderCache ); try{ - System.exit( - (Integer) res - .classLoader - .loadClass("cbt.Stage1") - .getMethod( - "run", - String[].class, File.class, File.class, BuildStage1Result.class, - Long.class, ConcurrentHashMap.class - ) - .invoke( - null, - (Object) args, new File(cache), new File(CBT_HOME), res, - start, classLoaderCache.hashMap - ) - ); + Integer exitCode = (Integer) res + .classLoader + .loadClass("cbt.Stage1") + .getMethod( + "run", String[].class, File.class, File.class, BuildStage1Result.class, Long.class, Map.class + ).invoke( + null, (Object) args, new File(cache), new File(CBT_HOME), res, start, classLoaderCache.hashMap + ); + + System.exit( exitCode ); } catch (java.lang.reflect.InvocationTargetException e) { throw unwrapInvocationTargetException(e); + } finally { + // This replaces the cache and should be thread-safe. + // For competing threads the last one wins with a consistent cache. + // So worst case, we loose some of the cache that's replaced. + classLoaderCacheHashMap = classLoaderCache.hashMap; } } diff --git a/stage1/ClassLoaderCache.scala b/stage1/ClassLoaderCache.scala index 2011562..af0970e 100644 --- a/stage1/ClassLoaderCache.scala +++ b/stage1/ClassLoaderCache.scala @@ -1,12 +1,12 @@ package cbt import java.net._ -import java.util.concurrent.ConcurrentHashMap +import java.util._ import collection.JavaConverters._ case class ClassLoaderCache( logger: Logger, - private[cbt] hashMap: ConcurrentHashMap[AnyRef,AnyRef] + private[cbt] hashMap: java.util.Map[AnyRef,AnyRef] ){ val cache = new KeyLockedLazyCache[ClassLoader]( hashMap, Some(logger) ) override def toString = ( diff --git a/stage1/ContextImplementation.scala b/stage1/ContextImplementation.scala index 6eb2e53..3b610c0 100644 --- a/stage1/ContextImplementation.scala +++ b/stage1/ContextImplementation.scala @@ -1,6 +1,5 @@ package cbt import java.io._ -import java.util.concurrent.ConcurrentHashMap import java.lang._ case class ContextImplementation( @@ -11,8 +10,8 @@ case class ContextImplementation( startCompat: Long, cbtHasChangedCompat: Boolean, scalaVersionOrNull: String, - persistentCache: ConcurrentHashMap[AnyRef,AnyRef], - transientCache: ConcurrentHashMap[AnyRef,AnyRef], + persistentCache: java.util.Map[AnyRef,AnyRef], + transientCache: java.util.Map[AnyRef,AnyRef], cache: File, cbtHome: File, cbtRootHome: File, diff --git a/stage1/KeyLockedLazyCache.scala b/stage1/KeyLockedLazyCache.scala index 2602523..2047b81 100644 --- a/stage1/KeyLockedLazyCache.scala +++ b/stage1/KeyLockedLazyCache.scala @@ -1,7 +1,5 @@ package cbt -import java.util.concurrent.ConcurrentHashMap - private[cbt] class LockableKey /** A hashMap that lazily computes values if needed during lookup. @@ -9,7 +7,7 @@ Locking occurs on the key, so separate keys can be looked up simultaneously without a deadlock. */ final private[cbt] class KeyLockedLazyCache[T <: AnyRef]( - val hashMap: ConcurrentHashMap[AnyRef,AnyRef], + val hashMap: java.util.Map[AnyRef,AnyRef], logger: Option[Logger] ){ def get( key: AnyRef, value: => T ): T = { diff --git a/stage1/PoorMansProfiler.scala b/stage1/PoorMansProfiler.scala index b7aa47d..4bc44ba 100644 --- a/stage1/PoorMansProfiler.scala +++ b/stage1/PoorMansProfiler.scala @@ -1,10 +1,10 @@ /* // temporary debugging tool package cbt -import java.util.concurrent.ConcurrentHashMap +import java.util._ import collection.JavaConversions._ object PoorMansProfiler{ - val entries = new ConcurrentHashMap[String, Long] + val entries = new HashMap[String, Long] def profile[T](name: String)(code: => T): T = { val before = System.currentTimeMillis if(!(entries containsKey name)){ diff --git a/stage1/Stage1.scala b/stage1/Stage1.scala index cd46d6b..2f8f960 100644 --- a/stage1/Stage1.scala +++ b/stage1/Stage1.scala @@ -1,7 +1,7 @@ package cbt import java.io._ -import java.util.concurrent.ConcurrentHashMap +import java.util._ import scala.collection.JavaConverters._ @@ -159,7 +159,7 @@ object Stage1{ cbtHome: File, buildStage1: BuildStage1Result, start: java.lang.Long, - persistentCache: ConcurrentHashMap[AnyRef,AnyRef] + persistentCache: java.util.Map[AnyRef,AnyRef] ): Int = { val args = Stage1ArgsParser(_args.toVector) val logger = new Logger(args.enabledLoggers, start) diff --git a/stage1/Stage1Lib.scala b/stage1/Stage1Lib.scala index 8fdde54..9b48409 100644 --- a/stage1/Stage1Lib.scala +++ b/stage1/Stage1Lib.scala @@ -9,7 +9,6 @@ import java.nio.file.attribute.FileTime import javax.tools._ import java.security._ import java.util.{Set=>_,Map=>_,List=>_,_} -import java.util.concurrent.ConcurrentHashMap import javax.xml.bind.annotation.adapters.HexBinaryAdapter // CLI interop diff --git a/stage1/cbt.scala b/stage1/cbt.scala index 985f619..22242d7 100644 --- a/stage1/cbt.scala +++ b/stage1/cbt.scala @@ -2,7 +2,6 @@ package cbt import java.io._ import java.nio.file._ import java.net._ -import java.util.concurrent.ConcurrentHashMap object `package`{ implicit class TypeInferenceSafeEquals[T](value: T){ diff --git a/stage1/resolver.scala b/stage1/resolver.scala index ff5ad68..1f94c7f 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -4,8 +4,6 @@ import java.nio.charset.StandardCharsets import java.net._ import java.io._ import scala.xml._ -import scala.concurrent._ -import scala.concurrent.duration._ trait DependencyImplementation extends Dependency{ implicit protected def logger: Logger diff --git a/stage2/Stage2.scala b/stage2/Stage2.scala index 260a46d..ab7b4fe 100644 --- a/stage2/Stage2.scala +++ b/stage2/Stage2.scala @@ -1,5 +1,6 @@ package cbt import java.io._ +import java.util._ object Stage2 extends Stage2Base{ def getBuild(context: Context) = { @@ -22,7 +23,7 @@ object Stage2 extends Stage2Base{ 0 } val task = args.args.lift( taskIndex ) - + val context: Context = ContextImplementation( args.cwd, args.cwd, @@ -32,7 +33,7 @@ object Stage2 extends Stage2Base{ args.cbtHasChanged, null, args.persistentCache, - new java.util.concurrent.ConcurrentHashMap, + new HashMap, args.cache, args.cbtHome, args.cbtHome, diff --git a/test/test.scala b/test/test.scala index 56ad3b1..ae6c301 100644 --- a/test/test.scala +++ b/test/test.scala @@ -1,9 +1,9 @@ package cbt package test -import java.util.concurrent.ConcurrentHashMap import java.io.File import java.nio.file._ import java.net.URL +import java.util.{Iterator=>_,_} import scala.concurrent._ import scala.concurrent.duration._ // micro framework @@ -115,8 +115,8 @@ object Main{ start, cbtHasChanged, null, - new ConcurrentHashMap[AnyRef,AnyRef], - new java.util.concurrent.ConcurrentHashMap[AnyRef,AnyRef], + new HashMap[AnyRef,AnyRef], + new HashMap[AnyRef,AnyRef], cache, cbtHome, cbtHome, -- cgit v1.2.3 From bee13ba7a4458482ce00a5c6bae4cd64328c4e5e Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Thu, 9 Feb 2017 02:29:44 +0000 Subject: memoize task results across classes within a single run --- compatibility/Dependency.java | 1 + plugins/essentials/DynamicOverrides.scala | 10 +-- plugins/scalajs/ScalaJsLib.scala | 2 +- stage1/Cache.scala | 14 ---- stage1/ContextImplementation.scala | 32 +++++----- stage1/MavenRepository.scala | 10 ++- stage1/Stage1.scala | 15 +++-- stage1/Stage1Lib.scala | 25 +++++++- stage1/cbt.scala | 4 +- stage1/logger.scala | 2 + stage1/resolver.scala | 102 ++++++++++++++++-------------- stage2/BasicBuild.scala | 29 ++------- stage2/BuildBuild.scala | 3 +- stage2/BuildDependency.scala | 3 + stage2/GitDependency.scala | 10 ++- stage2/Lib.scala | 4 +- stage2/PackageJars.scala | 9 +-- stage2/Stage2.scala | 4 +- stage2/ToolsTasks.scala | 1 + stage2/plugins/Dotty.scala | 5 +- test/test.scala | 5 +- 21 files changed, 148 insertions(+), 142 deletions(-) delete mode 100644 stage1/Cache.scala (limited to 'stage1/ContextImplementation.scala') diff --git a/compatibility/Dependency.java b/compatibility/Dependency.java index d491174..efb9214 100644 --- a/compatibility/Dependency.java +++ b/compatibility/Dependency.java @@ -3,6 +3,7 @@ import java.io.*; public interface Dependency{ public abstract String show(); + public abstract String moduleKey(); public abstract Boolean needsUpdateCompat(); public abstract Dependency[] dependenciesArray(); public abstract File[] dependencyClasspathArray(); diff --git a/plugins/essentials/DynamicOverrides.scala b/plugins/essentials/DynamicOverrides.scala index 4a3fe9e..1050f98 100644 --- a/plugins/essentials/DynamicOverrides.scala +++ b/plugins/essentials/DynamicOverrides.scala @@ -1,10 +1,12 @@ package cbt import cbt.eval.Eval trait DynamicOverrides extends BaseBuild{ - private val twitterEval = cached("eval"){ - new Eval{ - override lazy val impliedClassPath: List[String] = context.parentBuild.get.classpath.strings.toList//new ScalaCompilerDependency( context.cbtHasChanged, context.paths.mavenCache, scalaVersion ).classpath.strings.toList - override def classLoader = DynamicOverrides.this.getClass.getClassLoader + private val twitterEval = { + taskCache[DynamicOverrides]( "eval" ).memoize{ + new Eval{ + override lazy val impliedClassPath: List[String] = context.parentBuild.get.classpath.strings.toList//new ScalaCompilerDependency( context.cbtLastModified, context.paths.mavenCache, scalaVersion ).classpath.strings.toList + override def classLoader = DynamicOverrides.this.getClass.getClassLoader + } } } diff --git a/plugins/scalajs/ScalaJsLib.scala b/plugins/scalajs/ScalaJsLib.scala index 0355850..12c1c85 100644 --- a/plugins/scalajs/ScalaJsLib.scala +++ b/plugins/scalajs/ScalaJsLib.scala @@ -4,7 +4,7 @@ import java.io.File case class ScalaJsLib( scalaJsVersion: String, scalaVersion: String, cbtHasChanged: Boolean, classLoaderCache: ClassLoaderCache, mavenCache: File -)(implicit logger: Logger){ +)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]){ sealed trait ScalaJsOutputMode { def option: String def fileSuffix: String diff --git a/stage1/Cache.scala b/stage1/Cache.scala deleted file mode 100644 index a8036e5..0000000 --- a/stage1/Cache.scala +++ /dev/null @@ -1,14 +0,0 @@ -package cbt -/** -Caches exactly one value. -Is there a less boiler-platy way to achieve this, that doesn't -require creating an instance for each thing you want to cache? -*/ -class Cache[T]{ - private var value: Option[T] = None - def apply(value: => T) = this.synchronized{ - if(!this.value.isDefined) - this.value = Some(value) - this.value.get - } -} diff --git a/stage1/ContextImplementation.scala b/stage1/ContextImplementation.scala index 3b610c0..30db597 100644 --- a/stage1/ContextImplementation.scala +++ b/stage1/ContextImplementation.scala @@ -2,19 +2,19 @@ package cbt import java.io._ import java.lang._ -case class ContextImplementation( - projectDirectory: File, - cwd: File, - argsArray: Array[String], - enabledLoggersArray: Array[String], - startCompat: Long, - cbtHasChangedCompat: Boolean, - scalaVersionOrNull: String, - persistentCache: java.util.Map[AnyRef,AnyRef], - transientCache: java.util.Map[AnyRef,AnyRef], - cache: File, - cbtHome: File, - cbtRootHome: File, - compatibilityTarget: File, - parentBuildOrNull: BuildInterface -) extends Context \ No newline at end of file +class ContextImplementation( + val projectDirectory: File, + val cwd: File, + val argsArray: Array[String], + val enabledLoggersArray: Array[String], + val startCompat: Long, + val cbtHasChangedCompat: Boolean, + val scalaVersionOrNull: String, + val persistentCache: java.util.Map[AnyRef,AnyRef], + val transientCache: java.util.Map[AnyRef,AnyRef], + val cache: File, + val cbtHome: File, + val cbtRootHome: File, + val compatibilityTarget: File, + val parentBuildOrNull: BuildInterface +) extends Context diff --git a/stage1/MavenRepository.scala b/stage1/MavenRepository.scala index 4184d2d..2ac7064 100644 --- a/stage1/MavenRepository.scala +++ b/stage1/MavenRepository.scala @@ -1,9 +1,13 @@ package cbt import java.io._ import java.net._ -case class MavenResolver( cbtHasChanged: Boolean, mavenCache: File, urls: URL* ){ - def bind( dependencies: MavenDependency* )(implicit logger: Logger): Seq[BoundMavenDependency] +case class MavenResolver( + cbtHasChanged: Boolean, mavenCache: File, urls: URL* +)( + implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef] +){ + def bind( dependencies: MavenDependency* ): Seq[BoundMavenDependency] = dependencies.map( BoundMavenDependency(cbtHasChanged,mavenCache,_,urls.to) ).to - def bindOne( dependency: MavenDependency )(implicit logger: Logger): BoundMavenDependency + def bindOne( dependency: MavenDependency ): BoundMavenDependency = BoundMavenDependency( cbtHasChanged, mavenCache, dependency, urls.to ) } diff --git a/stage1/Stage1.scala b/stage1/Stage1.scala index f0540b4..27c6402 100644 --- a/stage1/Stage1.scala +++ b/stage1/Stage1.scala @@ -44,6 +44,8 @@ case class Stage2Args( cache: File, cbtHome: File, compatibilityTarget: File +)( + implicit val transientCache: java.util.Map[AnyRef,AnyRef] ){ val ClassLoaderCache( logger, persistentCache ) = classLoaderCache } @@ -60,7 +62,7 @@ object Stage1{ ClassLoaderCache( logger, context.persistentCache ), context.cbtHome, context.cache - ) + )( context.transientCache ) classLoader .loadClass("cbt.Stage2") @@ -75,7 +77,7 @@ object Stage1{ def buildStage2( buildStage1: BuildStage1Result, classLoaderCache: ClassLoaderCache, cbtHome: File, cache: File - ): (Boolean, ClassLoader) = { + )(implicit transientCache: java.util.Map[AnyRef,AnyRef]): (Boolean, ClassLoader) = { import classLoaderCache.logger val lib = new Stage1Lib(logger) @@ -86,12 +88,12 @@ object Stage1{ val stage2sourceFiles = ( stage2.listFiles ++ (stage2 ++ "/plugins").listFiles ).toVector.filter(_.isFile).filter(_.toString.endsWith(".scala")) - + val cbtHasChanged = buildStage1.changed || lib.needsUpdate(stage2sourceFiles, stage2StatusFile) val cls = this.getClass.getClassLoader.loadClass("cbt.NailgunLauncher") - - val cbtDependency = CbtDependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, stage2Target, new File(buildStage1.compatibilityClasspath)) + + val cbtDependency = new CbtDependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, stage2Target, new File(buildStage1.compatibilityClasspath)) logger.stage1("Compiling stage2 if necessary") compile( @@ -102,7 +104,7 @@ object Stage1{ mavenCache, Seq("-deprecation","-feature","-unchecked"), classLoaderCache, zincVersion = constants.zincVersion, scalaVersion = constants.scalaVersion - ) + )(transientCache) logger.stage1(s"calling CbtDependency.classLoader") if( cbtHasChanged && classLoaderCache.cache.containsKey( cbtDependency.classpath.string ) ) { @@ -163,6 +165,7 @@ object Stage1{ ): Int = { val args = Stage1ArgsParser(_args.toVector) val logger = new Logger(args.enabledLoggers, start) + implicit val transientCache: java.util.Map[AnyRef,AnyRef] = new java.util.HashMap logger.stage1(s"Stage1 start") val classLoaderCache = ClassLoaderCache( logger, persistentCache ) diff --git a/stage1/Stage1Lib.scala b/stage1/Stage1Lib.scala index 296581c..f0bb588 100644 --- a/stage1/Stage1Lib.scala +++ b/stage1/Stage1Lib.scala @@ -200,8 +200,9 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ classLoaderCache: ClassLoaderCache, zincVersion: String, scalaVersion: String + )( + implicit transientCache: java.util.Map[AnyRef, AnyRef] ): Option[File] = { - val classpath = Dependencies(dependencies).classpath val cp = classpath.string if(classpath.files.isEmpty) @@ -447,4 +448,26 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ else cache.cache.get( a.classpath.string, cl ).asInstanceOf[ClassLoader] } + +} + +import scala.reflect._ +import scala.language.existentials +case class PerClassCache(cache: java.util.Map[AnyRef,AnyRef], moduleKey: String)(implicit logger: Logger){ + def apply[D <: Dependency: ClassTag](key: AnyRef): MethodCache[D] = new MethodCache[D](key) + case class MethodCache[D <: Dependency: ClassTag](key: AnyRef){ + def memoize[T <: AnyRef](task: => T): T = { + val fullKey = (classTag[D].runtimeClass, moduleKey, key) + logger.transientCache("fetching key"+fullKey) + if( cache.containsKey(fullKey) ){ + logger.transientCache("found key"+fullKey) + cache.get(fullKey).asInstanceOf[T] + } else{ + val value = task + logger.transientCache("put key"+fullKey) + cache.put( fullKey, value ) + value + } + } + } } diff --git a/stage1/cbt.scala b/stage1/cbt.scala index 22242d7..01c9303 100644 --- a/stage1/cbt.scala +++ b/stage1/cbt.scala @@ -62,7 +62,7 @@ object `package`{ def classLoaderCache: ClassLoaderCache = new ClassLoaderCache( logger, persistentCache ) def cbtDependency = { import paths._ - CbtDependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, stage2Target, compatibilityTarget) + new CbtDependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, stage2Target, compatibilityTarget)(logger, transientCache) } def args: Seq[String] = argsArray.to def enabledLoggers: Set[String] = enabledLoggersArray.to @@ -79,7 +79,7 @@ object `package`{ scalaVersion: Option[String] = scalaVersion, cbtHome: File = cbtHome, parentBuild: Option[BuildInterface] = None - ): Context = ContextImplementation( + ): Context = new ContextImplementation( projectDirectory, cwd, args.to, diff --git a/stage1/logger.scala b/stage1/logger.scala index effdc35..8c8431a 100644 --- a/stage1/logger.scala +++ b/stage1/logger.scala @@ -41,6 +41,7 @@ case class Logger(enabledLoggers: Set[String], start: Long) { final def git(msg: => String) = log(names.git, msg) final def pom(msg: => String) = log(names.pom, msg) final def dynamic(msg: => String) = log(names.dynamic, msg) + final def transientCache(msg: => String) = log(names.transientCache, msg) private object names{ val stage1 = "stage1" @@ -54,6 +55,7 @@ case class Logger(enabledLoggers: Set[String], start: Long) { val pom = "pom" val git = "git" val dynamic = "dynamic" + val transientCache = "transientCache" } private def logUnguarded(name: String, msg: => String) = { diff --git a/stage1/resolver.scala b/stage1/resolver.scala index f4af73e..8e46135 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -8,6 +8,15 @@ import scala.xml._ trait DependencyImplementation extends Dependency{ implicit protected def logger: Logger protected def lib = new Stage1Lib(logger) + implicit protected def transientCache: java.util.Map[AnyRef,AnyRef] + + /** key used by taskCache to identify different objects that represent the same logical module */ + protected def moduleKey: String + /** + caches given value in context keyed with given key and projectDirectory + the context is fresh on every complete run of cbt + */ + protected lazy val taskCache = new PerClassCache(transientCache, moduleKey) /** CAREFUL: this is never allowed to return true for the same dependency more than @@ -101,11 +110,11 @@ trait DependencyImplementation extends Dependency{ ) def dependencies: Seq[Dependency] - private object transitiveDependenciesCache extends Cache[Seq[Dependency]] /** return dependencies in order of linearized dependence. this is a bit tricky. */ - def transitiveDependencies: Seq[Dependency] = transitiveDependenciesCache{ - lib.transitiveDependencies(this) - } + def transitiveDependencies: Seq[Dependency] = + taskCache[DependencyImplementation]( "transitiveDependencies" ).memoize{ + lib.transitiveDependencies(this) + } override def show: String = this.getClass.getSimpleName // ========== debug ========== @@ -113,65 +122,63 @@ trait DependencyImplementation extends Dependency{ } // TODO: all this hard codes the scala version, needs more flexibility -class ScalaCompilerDependency(cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-compiler",version, Classifier.none), Seq(mavenCentral)) -class ScalaLibraryDependency (cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-library",version, Classifier.none), Seq(mavenCentral)) -class ScalaReflectDependency (cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-reflect",version, Classifier.none), Seq(mavenCentral)) +class ScalaCompilerDependency(cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-compiler",version, Classifier.none), Seq(mavenCentral)) +class ScalaLibraryDependency (cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-library",version, Classifier.none), Seq(mavenCentral)) +class ScalaReflectDependency (cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-reflect",version, Classifier.none), Seq(mavenCentral)) -case class ScalaDependencies(cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit val logger: Logger) extends DependencyImplementation{ sd => - override final val needsUpdate = false - def targetClasspath = ClassPath() - def exportedClasspath = ClassPath() - def dependencies = Seq( +class ScalaDependencies(cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends Dependencies( + Seq( new ScalaCompilerDependency(cbtHasChanged, mavenCache, version), new ScalaLibraryDependency(cbtHasChanged, mavenCache, version), new ScalaReflectDependency(cbtHasChanged, mavenCache, version) ) -} +) -case class BinaryDependency( paths: Seq[File], dependencies: Seq[Dependency] )(implicit val logger: Logger) extends DependencyImplementation{ +case class BinaryDependency( paths: Seq[File], dependencies: Seq[Dependency] )(implicit val logger: Logger, val transientCache: java.util.Map[AnyRef,AnyRef]) extends DependencyImplementation{ + assert(paths.nonEmpty) def exportedClasspath = ClassPath(paths) override def needsUpdate = false def targetClasspath = exportedClasspath + def moduleKey = this.getClass.getName ++ "(" ++ paths.mkString(", ") ++ ")" } /** Allows to easily assemble a bunch of dependencies */ -case class Dependencies( dependencies: Seq[Dependency] )(implicit val logger: Logger) extends DependencyImplementation{ +case class Dependencies( dependencies: Seq[Dependency] )(implicit val logger: Logger, val transientCache: java.util.Map[AnyRef,AnyRef]) extends DependencyImplementation{ override def needsUpdate = dependencies.exists(_.needsUpdate) override def exportedClasspath = ClassPath() override def targetClasspath = ClassPath() + def moduleKey = this.getClass.getName ++ "(" ++ dependencies.map(_.moduleKey).mkString(", ") ++ ")" } -case class Stage1Dependency(cbtHasChanged: Boolean, mavenCache: File, nailgunTarget: File, stage1Target: File, compatibilityTarget: File)(implicit val logger: Logger) extends DependencyImplementation{ - override def needsUpdate = false - override def targetClasspath = exportedClasspath - override def exportedClasspath = ClassPath( Seq(nailgunTarget, stage1Target) ) - val compatibilityDependency = CompatibilityDependency(cbtHasChanged, compatibilityTarget) - override def dependencies = Seq( - compatibilityDependency +class Stage1Dependency(cbtHasChanged: Boolean, mavenCache: File, nailgunTarget: File, stage1Target: File, compatibilityTarget: File)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BinaryDependency( + Seq(nailgunTarget, stage1Target), + Seq( + new CompatibilityDependency(cbtHasChanged, compatibilityTarget) ) ++ MavenResolver(cbtHasChanged,mavenCache,mavenCentral).bind( MavenDependency("org.scala-lang","scala-library",constants.scalaVersion), MavenDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,constants.scalaXmlVersion) ) +){ + val compatibilityDependency = new CompatibilityDependency(cbtHasChanged, compatibilityTarget) + } -case class CompatibilityDependency(cbtHasChanged: Boolean, compatibilityTarget: File)(implicit val logger: Logger) extends DependencyImplementation{ - override def needsUpdate = false - override def targetClasspath = exportedClasspath - override def exportedClasspath = ClassPath( Seq(compatibilityTarget) ) - override def dependencies = Seq() -} -case class CbtDependency(cbtHasChanged: Boolean, mavenCache: File, nailgunTarget: File, stage1Target: File, stage2Target: File, compatibilityTarget: File)(implicit val logger: Logger) extends DependencyImplementation{ - override def needsUpdate = false - override def targetClasspath = exportedClasspath - override def exportedClasspath = ClassPath( Seq( stage2Target ) ) - val stage1Dependency = Stage1Dependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, compatibilityTarget) - override def dependencies = Seq( - stage1Dependency + +class CompatibilityDependency(cbtHasChanged: Boolean, compatibilityTarget: File)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BinaryDependency( + Seq(compatibilityTarget), Nil +) + +class CbtDependency(cbtHasChanged: Boolean, mavenCache: File, nailgunTarget: File, stage1Target: File, stage2Target: File, compatibilityTarget: File)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BinaryDependency( + Seq( stage2Target ), + Seq( + new Stage1Dependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, compatibilityTarget) ) ++ MavenResolver(cbtHasChanged, mavenCache,mavenCentral).bind( MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0"), MavenDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r") ) +){ + val stage1Dependency = new Stage1Dependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, compatibilityTarget) } case class Classifier(name: Option[String]) @@ -185,7 +192,9 @@ abstract class DependenciesProxy{ } class BoundMavenDependencies( cbtHasChanged: Boolean, mavenCache: File, urls: Seq[URL], mavenDependencies: Seq[MavenDependency] -)(implicit logger: Logger) extends Dependencies( +)( + implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef] +) extends Dependencies( mavenDependencies.map( BoundMavenDependency(cbtHasChanged,mavenCache,_,urls) ) ) case class MavenDependency( @@ -201,7 +210,10 @@ object MavenDependency{ // FIXME: take MavenResolver instead of mavenCache and repositories separately case class BoundMavenDependency( cbtHasChanged: Boolean, mavenCache: File, mavenDependency: MavenDependency, repositories: Seq[URL] -)(implicit val logger: Logger) extends ArtifactInfo with DependencyImplementation{ +)( + implicit val logger: Logger, val transientCache: java.util.Map[AnyRef,AnyRef] +) extends ArtifactInfo with DependencyImplementation{ + def moduleKey = this.getClass.getName ++ "(" ++ mavenDependency.serialize ++ ")" val MavenDependency( groupId, artifactId, version, classifier ) = mavenDependency assert( Option(groupId).collect{ @@ -251,17 +263,11 @@ case class BoundMavenDependency( StandardCharsets.UTF_8 ).mkString("\n").split(" ").head.trim } - - private object jarSha1Cache extends Cache[String] - def jarSha1: String = jarSha1Cache{ resolveHash("jar", true) } - - private object pomSha1Cache extends Cache[String] - def pomSha1: String = pomSha1Cache{ resolveHash("pom", false) } - private object jarCache extends Cache[File] - def jar: File = jarCache{ resolve("jar", Some(jarSha1), true) } - private object pomCache extends Cache[File] - def pom: File = pomCache{ resolve("pom", Some(pomSha1), false) } + def jarSha1: String = taskCache[BoundMavenDependency]("jarSha1").memoize{ resolveHash("jar", true) } + def pomSha1: String = taskCache[BoundMavenDependency]("pomSha1").memoize{ resolveHash("pom", false) } + def jar: File = taskCache[BoundMavenDependency]("jar").memoize{ resolve("jar", Some(jarSha1), true) } + def pom: File = taskCache[BoundMavenDependency]("pom").memoize{ resolve("pom", Some(pomSha1), false) } private def pomXml = XML.loadFile(pom.string) // ========== pom traversal ========== @@ -278,7 +284,7 @@ case class BoundMavenDependency( (parent \ "version").text ), repositories - )(logger) + ) }.flatMap(_.transitivePom) :+ this } diff --git a/stage2/BasicBuild.scala b/stage2/BasicBuild.scala index 8b4a3a5..2fd34c7 100644 --- a/stage2/BasicBuild.scala +++ b/stage2/BasicBuild.scala @@ -9,6 +9,8 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with Trigge // will create new instances given the context, which means operations in the // overrides will happen multiple times and if they are not idempotent stuff likely breaks def context: Context + def moduleKey: String = "BaseBuild("+projectDirectory.string+")" + implicit def transientCache: java.util.Map[AnyRef,AnyRef] = context.transientCache // library available to builds implicit protected final val logger: Logger = context.logger @@ -59,7 +61,7 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with Trigge // FIXME: this should probably be removed Resolver( mavenCentral ).bind( "org.scala-lang" % "scala-library" % scalaVersion - ) :+ BinaryDependency(localJars, Nil) + ) ++ ( if(localJars.nonEmpty) Seq( BinaryDependency(localJars, Nil) ) else Nil ) // ========== paths ========== final private val defaultSourceDirectory = projectDirectory ++ "/src" @@ -135,15 +137,13 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with Trigge "-unchecked" ) - private object needsUpdateCache extends Cache[Boolean] - def needsUpdate: Boolean = needsUpdateCache( + def needsUpdate: Boolean = taskCache[BaseBuild]("needsUpdate").memoize[java.lang.Boolean]( context.cbtHasChanged || lib.needsUpdate( sourceFiles, compileStatusFile ) || transitiveDependencies.filterNot(_ == context.parentBuild).exists(_.needsUpdate) ) - private object compileCache extends Cache[Option[File]] - def compile: Option[File] = compileCache{ + def compile: Option[File] = taskCache[BaseBuild]("compile").memoize{ lib.compile( context.cbtHasChanged, needsUpdate || context.parentBuild.map(_.needsUpdate).getOrElse(false), @@ -153,7 +153,6 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with Trigge ) } - def mainClasses: Seq[Class[_]] = compile.toSeq.flatMap( lib.mainClasses( _, classLoader(classLoaderCache) ) ) def runClass: Option[String] = lib.runClass( mainClasses ).map( _.getName ) @@ -264,24 +263,6 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with Trigge def finalBuild: BuildInterface = this override def show = this.getClass.getSimpleName ++ "(" ++ projectDirectory.string ++ ")" - // TODO: allow people not provide the method name, maybe via macro - // TODO: pull this out into lib - /** - caches given value in context keyed with given key and projectDirectory - the context is fresh on every complete run of cbt - */ - def cached[T <: AnyRef](name: String)(task: => T): T = { - val cache = context.transientCache - val key = (projectDirectory,name) - if( cache.containsKey(key) ){ - cache.get(key).asInstanceOf[T] - } else{ - val value = task - cache.put( key, value ) - value - } - } - // a method that can be called only to trigger any side-effects final def `void` = () } diff --git a/stage2/BuildBuild.scala b/stage2/BuildBuild.scala index cf515bb..1b05214 100644 --- a/stage2/BuildBuild.scala +++ b/stage2/BuildBuild.scala @@ -32,8 +32,7 @@ trait BuildBuildWithoutEssentials extends BaseBuild{ override def dependencies = super.dependencies :+ context.cbtDependency def managedBuildDirectory: java.io.File = lib.realpath( projectDirectory.parent ) - private object managedBuildCache extends Cache[BuildInterface] - def managedBuild = managedBuildCache{ + def managedBuild = taskCache[BuildBuildWithoutEssentials]("managedBuild").memoize{ val managedBuildFile = projectDirectory++"/build.scala" logger.composition("Loading build at " ++ managedBuildDirectory.toString) val build = ( diff --git a/stage2/BuildDependency.scala b/stage2/BuildDependency.scala index 197a7a1..4b4fdc1 100644 --- a/stage2/BuildDependency.scala +++ b/stage2/BuildDependency.scala @@ -16,9 +16,12 @@ trait TriggerLoop extends DependencyImplementation{ } /** You likely want to use the factory method in the BasicBuild class instead of this. */ final case class DirectoryDependency(context: Context) extends TriggerLoop{ + override def toString = show override def show = this.getClass.getSimpleName ++ "(" ++ context.projectDirectory.string ++ ")" + def moduleKey = this.getClass.getName ++ "("+context.projectDirectory.string+")" lazy val logger = context.logger override lazy val lib: Lib = new Lib(logger) + def transientCache = context.transientCache private lazy val root = lib.loadRoot( context.copy(args=Seq()) ) lazy val build = root.finalBuild def exportedClasspath = ClassPath() diff --git a/stage2/GitDependency.scala b/stage2/GitDependency.scala index 650fd09..e27eff9 100644 --- a/stage2/GitDependency.scala +++ b/stage2/GitDependency.scala @@ -15,15 +15,14 @@ case class GitDependency( )(implicit val logger: Logger, classLoaderCache: ClassLoaderCache, context: Context ) extends DependencyImplementation{ import GitDependency._ override def lib = new Lib(logger) - + def moduleKey = this.getClass.getName ++ "(" ++ url ++ subDirectory.map("/" ++ _).getOrElse("") ++ "#" ++ ref ++ ")" + def transientCache = context.transientCache // TODO: add support for authentication via ssh and/or https // See http://www.codeaffine.com/2014/12/09/jgit-authentication/ private val GitUrl( _, domain, path ) = url private val credentialsFile = context.projectDirectory ++ "/git.login" - private object checkoutCache extends Cache[File] - private def authenticate(_git: CloneCommand) = if(!credentialsFile.exists){ _git @@ -36,7 +35,7 @@ case class GitDependency( _git.setCredentialsProvider( new UsernamePasswordCredentialsProvider(user, password) ) } - def checkout: File = checkoutCache{ + def checkout: File = taskCache[GitDependency]("checkout").memoize{ val checkoutDirectory = context.cache ++ s"/git/$domain/$path/$ref" val _git = if(checkoutDirectory.exists){ logger.git(s"Found existing checkout of $url#$ref in $checkoutDirectory") @@ -65,8 +64,7 @@ case class GitDependency( assert( actualRef == ref, s"actual ref '$actualRef' does not match expected ref '$ref'") checkoutDirectory } - private object dependencyCache extends Cache[DependencyImplementation] - def dependency = dependencyCache{ + def dependency = taskCache[GitDependency]("dependency").memoize{ DirectoryDependency( context.copy( projectDirectory = checkout ++ subDirectory.map("/" ++ _).getOrElse("") diff --git a/stage2/Lib.scala b/stage2/Lib.scala index 769cd97..c570ca3 100644 --- a/stage2/Lib.scala +++ b/stage2/Lib.scala @@ -86,7 +86,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ compileArgs: Seq[String], classLoaderCache: ClassLoaderCache, mavenCache: File - ): Option[File] = { + )(implicit transientCache: java.util.Map[AnyRef,AnyRef]): Option[File] = { if(sourceFiles.isEmpty){ None } else { @@ -101,7 +101,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ runMain( "scala.tools.nsc.ScalaDoc", args, - ScalaDependencies(cbtHasChanged,mavenCache,scalaVersion)(logger).classLoader(classLoaderCache) + new ScalaDependencies(cbtHasChanged,mavenCache,scalaVersion).classLoader(classLoaderCache) ) } lib.jarFile( diff --git a/stage2/PackageJars.scala b/stage2/PackageJars.scala index ff89284..3ecceb2 100644 --- a/stage2/PackageJars.scala +++ b/stage2/PackageJars.scala @@ -10,18 +10,15 @@ trait PackageJars extends BaseBuild with ArtifactInfo{ Seq(() => jar, () => docJar, () => srcJar) )( _() ).flatten - private object cacheJarBasicBuild extends Cache[Option[File]] - def jar: Option[File] = cacheJarBasicBuild{ + def jar: Option[File] = taskCache[PackageJars]("jar").memoize{ compile.flatMap( lib.jar( artifactId, scalaMajorVersion, version, _, jarTarget ) ) } - private object cacheSrcJarBasicBuild extends Cache[Option[File]] - def srcJar: Option[File] = cacheSrcJarBasicBuild{ + def srcJar: Option[File] = taskCache[PackageJars]("srcJar").memoize{ lib.srcJar( sourceFiles, artifactId, scalaMajorVersion, version, scalaTarget ) } - private object cacheDocBasicBuild extends Cache[Option[File]] - def docJar: Option[File] = cacheDocBasicBuild{ + def docJar: Option[File] = taskCache[PackageJars]("docJar").memoize{ lib.docJar( context.cbtHasChanged, scalaVersion, sourceFiles, compileClasspath, docTarget, diff --git a/stage2/Stage2.scala b/stage2/Stage2.scala index ab7b4fe..2884ddb 100644 --- a/stage2/Stage2.scala +++ b/stage2/Stage2.scala @@ -24,7 +24,7 @@ object Stage2 extends Stage2Base{ } val task = args.args.lift( taskIndex ) - val context: Context = ContextImplementation( + val context: Context = new ContextImplementation( args.cwd, args.cwd, args.args.drop( taskIndex +1 ).toArray, @@ -33,7 +33,7 @@ object Stage2 extends Stage2Base{ args.cbtHasChanged, null, args.persistentCache, - new HashMap, + args.transientCache, args.cache, args.cbtHome, args.cbtHome, diff --git a/stage2/ToolsTasks.scala b/stage2/ToolsTasks.scala index b92cb7a..839780a 100644 --- a/stage2/ToolsTasks.scala +++ b/stage2/ToolsTasks.scala @@ -15,6 +15,7 @@ class ToolsTasks( import paths._ private def Resolver( urls: URL* ) = MavenResolver(cbtHasChanged,mavenCache,urls: _*) implicit val logger: Logger = lib.logger + implicit val transientCache: java.util.Map[AnyRef,AnyRef] = new java.util.HashMap def createMain: Unit = lib.createMain( cwd ) def createBuild: Unit = lib.createBuild( cwd ) def gui = NailgunLauncher.main(Array( diff --git a/stage2/plugins/Dotty.scala b/stage2/plugins/Dotty.scala index 8671fb6..fe949a3 100644 --- a/stage2/plugins/Dotty.scala +++ b/stage2/plugins/Dotty.scala @@ -14,8 +14,7 @@ trait Dotty extends BaseBuild{ context.classLoaderCache, dottyVersion = dottyVersion ) - private object compileCache extends Cache[Option[File]] - override def compile: Option[File] = compileCache{ + override def compile: Option[File] = taskCache[Dotty]("compile").memoize{ dottyLib.compile( needsUpdate || context.parentBuild.map(_.needsUpdate).getOrElse(false), sourceFiles, compileTarget, compileStatusFile, compileClasspath, @@ -41,7 +40,7 @@ class DottyLib( mavenCache: File, classLoaderCache: ClassLoaderCache, dottyVersion: String -){ +)(implicit transientCache: java.util.Map[AnyRef,AnyRef]){ val lib = new Lib(logger) import lib._ diff --git a/test/test.scala b/test/test.scala index ae6c301..332b61e 100644 --- a/test/test.scala +++ b/test/test.scala @@ -104,10 +104,11 @@ object Main{ val cache = cbtHome ++ "/cache" val mavenCache = cache ++ "/maven" val cbtHasChanged = true - def Resolver(urls: URL*) = MavenResolver(cbtHasChanged, mavenCache, urls: _*) + implicit val transientCache: java.util.Map[AnyRef,AnyRef] = new java.util.HashMap + def Resolver(urls: URL*) = MavenResolver(cbtLastModified, mavenCache, urls: _*) { - val noContext = ContextImplementation( + val noContext = new ContextImplementation( cbtHome ++ "/test/nothing", cbtHome, Array(), -- cgit v1.2.3 From e8673866b79f7473391dcee26243eee80d5d3cb6 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Thu, 9 Feb 2017 21:20:11 -0500 Subject: idempotent change propagation using lastModified instead of a non-idempotent needsUpdate flag this fixes a bug where dependees would not be rebuilt if cbt exited or was killed after dependencies were already rebuilt. --- circle.yml | 2 +- compatibility/Context.java | 32 ++++- compatibility/Dependency.java | 16 ++- compatibility/IncompatibleCbtVersionException.java | 10 ++ libraries/eval/build/build.scala | 2 +- nailgun_launcher/BuildStage1Result.java | 8 +- nailgun_launcher/CbtURLClassLoader.java | 8 +- nailgun_launcher/ClassLoaderCache.java | 66 ++++++++++ nailgun_launcher/EarlyDependencies.java | 140 +++++++++++---------- nailgun_launcher/JavaCache.java | 40 ------ nailgun_launcher/NailgunLauncher.java | 80 +++++++----- nailgun_launcher/Stage0Lib.java | 53 ++++++-- plugins/scalajs/ScalaJsBuild.scala | 2 +- plugins/scalajs/ScalaJsLib.scala | 4 +- plugins/scalatest/ScalaTest.scala | 2 +- plugins/wartremover/WartRemover.scala | 2 +- stage1/CbtPaths.scala | 2 + stage1/ClassLoaderCache.scala | 24 ---- stage1/ContextImplementation.scala | 40 +++--- stage1/MavenRepository.scala | 6 +- stage1/Stage1.scala | 100 +++++++-------- stage1/Stage1Lib.scala | 81 ++++++------ stage1/cbt.scala | 23 ++-- stage1/resolver.scala | 102 +++++++-------- stage2/BasicBuild.scala | 21 ++-- stage2/BuildDependency.scala | 2 +- stage2/GitDependency.scala | 2 +- stage2/Lib.scala | 6 +- stage2/PackageJars.scala | 4 +- stage2/Stage2.scala | 4 +- stage2/ToolsStage2.scala | 2 +- stage2/ToolsTasks.scala | 26 ++-- stage2/plugins/Dotty.scala | 44 +++---- test/test.scala | 4 +- 34 files changed, 544 insertions(+), 416 deletions(-) create mode 100644 compatibility/IncompatibleCbtVersionException.java create mode 100644 nailgun_launcher/ClassLoaderCache.java delete mode 100644 nailgun_launcher/JavaCache.java delete mode 100644 stage1/ClassLoaderCache.scala (limited to 'stage1/ContextImplementation.scala') diff --git a/circle.yml b/circle.yml index 1615ad5..9de5b42 100644 --- a/circle.yml +++ b/circle.yml @@ -1,6 +1,6 @@ machine: java: - version: oraclejdk7 + version: oraclejdk8 dependencies: cache_directories: diff --git a/compatibility/Context.java b/compatibility/Context.java index a7af740..afd0b15 100644 --- a/compatibility/Context.java +++ b/compatibility/Context.java @@ -1,21 +1,43 @@ package cbt; import java.io.*; import java.util.*; +import java.util.concurrent.*; // TODO: try to reduce the number of members -public abstract class Context{ +public interface Context{ + // recently added methods that needs default values for old versions to work + public default long cbtLastModified(){ + throw new IncompatibleCbtVersionException("You need to define method cbtLastModified."); + }; + public default Map persistentCache(){ + throw new IncompatibleCbtVersionException("You need to define method persistentCache."); + }; + public default Map transientCache(){ + throw new IncompatibleCbtVersionException("You need to define method transientCache."); + }; + public default long start(){ + throw new IncompatibleCbtVersionException("You need to define method start."); + }; + + // methods that exist for longer which every CBT version in use should have by now, no default values needed public abstract File projectDirectory(); public abstract File cwd(); // REPLACE by something that allows to run cbt on some other directly public abstract String[] argsArray(); // replace this by https://github.com/cvogt/cbt/issues/172 ? public abstract String[] enabledLoggersArray(); - public abstract Long startCompat(); - public abstract Boolean cbtHasChangedCompat(); public abstract String scalaVersionOrNull(); // needed to propagate scalaVersion to dependendee builds - public abstract Map persistentCache(); - public abstract Map transientCache(); public abstract File cache(); public abstract File cbtHome(); public abstract File cbtRootHome(); // REMOVE public abstract File compatibilityTarget(); // maybe replace this with search in the classloader for it? public abstract BuildInterface parentBuildOrNull(); + + // deprecated methods + @java.lang.Deprecated + public abstract Long startCompat(); + @java.lang.Deprecated + public abstract Boolean cbtHasChangedCompat(); + @java.lang.Deprecated + public abstract ConcurrentHashMap permanentKeys(); + @java.lang.Deprecated + public abstract ConcurrentHashMap permanentClassLoaders(); } diff --git a/compatibility/Dependency.java b/compatibility/Dependency.java index efb9214..1f719c2 100644 --- a/compatibility/Dependency.java +++ b/compatibility/Dependency.java @@ -2,10 +2,22 @@ package cbt; import java.io.*; public interface Dependency{ + // recently added methods that needs default values for old versions to work + public default String moduleKey(){ + throw new IncompatibleCbtVersionException("You need to define method moduleKey."); + }; + public default long lastModified(){ + throw new IncompatibleCbtVersionException("You need to define method lastModified."); + }; + + // methods that exist for longer which every CBT version in use should have by now, no default values needed public abstract String show(); - public abstract String moduleKey(); - public abstract Boolean needsUpdateCompat(); public abstract Dependency[] dependenciesArray(); public abstract File[] dependencyClasspathArray(); public abstract File[] exportedClasspathArray(); + + // deprecated methods + @java.lang.Deprecated + public abstract boolean needsUpdateCompat(); } + diff --git a/compatibility/IncompatibleCbtVersionException.java b/compatibility/IncompatibleCbtVersionException.java new file mode 100644 index 0000000..dee50fb --- /dev/null +++ b/compatibility/IncompatibleCbtVersionException.java @@ -0,0 +1,10 @@ +package cbt; + +public class IncompatibleCbtVersionException extends RuntimeException{ + public IncompatibleCbtVersionException( String msg, Throwable parent ){ + super( msg, parent ); + } + public IncompatibleCbtVersionException( String msg ){ + super( msg ); + } +} diff --git a/libraries/eval/build/build.scala b/libraries/eval/build/build.scala index 8dcaabd..7135d3f 100644 --- a/libraries/eval/build/build.scala +++ b/libraries/eval/build/build.scala @@ -2,7 +2,7 @@ import cbt._ class Build(val context: Context) extends BaseBuild{ outer => override def dependencies = super.dependencies :+ - new ScalaCompilerDependency( context.cbtHasChanged, context.paths.mavenCache, scalaVersion ) + new ScalaCompilerDependency( context.cbtLastModified, context.paths.mavenCache, scalaVersion ) override def test: Option[ExitCode] = Some{ new BasicBuild(context.copy(projectDirectory = projectDirectory ++ "/test")) with ScalaTest{ diff --git a/nailgun_launcher/BuildStage1Result.java b/nailgun_launcher/BuildStage1Result.java index 312871d..64a660d 100644 --- a/nailgun_launcher/BuildStage1Result.java +++ b/nailgun_launcher/BuildStage1Result.java @@ -1,12 +1,14 @@ package cbt; public class BuildStage1Result{ - public Boolean changed; + public long start; + public long stage1LastModified; public ClassLoader classLoader; public String stage1Classpath; public String nailgunClasspath; public String compatibilityClasspath; - public BuildStage1Result( Boolean changed, ClassLoader classLoader, String stage1Classpath, String nailgunClasspath, String compatibilityClasspath ){ - this.changed = changed; + public BuildStage1Result( long start, long stage1LastModified, ClassLoader classLoader, String stage1Classpath, String nailgunClasspath, String compatibilityClasspath ){ + this.start = start; + this.stage1LastModified = stage1LastModified; this.classLoader = classLoader; this.stage1Classpath = stage1Classpath; this.nailgunClasspath = nailgunClasspath; diff --git a/nailgun_launcher/CbtURLClassLoader.java b/nailgun_launcher/CbtURLClassLoader.java index 43d07f4..fac7050 100644 --- a/nailgun_launcher/CbtURLClassLoader.java +++ b/nailgun_launcher/CbtURLClassLoader.java @@ -15,7 +15,7 @@ public class CbtURLClassLoader extends java.net.URLClassLoader{ + "\n)" ); } - JavaCache cache = new JavaCache( new ConcurrentHashMap() ); + ConcurrentHashMap cache = new ConcurrentHashMap(); public Class loadClass(String name) throws ClassNotFoundException{ Class _class = super.loadClass(name); if(_class == null) throw new ClassNotFoundException(name); @@ -24,11 +24,11 @@ public class CbtURLClassLoader extends java.net.URLClassLoader{ public Class loadClass(String name, Boolean resolve) throws ClassNotFoundException{ //System.out.println("loadClass("+name+") on \n"+this); synchronized( cache ){ - if(!cache.contains(name)) + if(!cache.containsKey(name)) try{ - cache.put(super.loadClass(name, resolve), name); + cache.put(name, super.loadClass(name, resolve)); } catch (ClassNotFoundException e){ - cache.put(Object.class, name); + cache.put(name, Object.class); } Class _class = cache.get(name); if(_class == Object.class){ diff --git a/nailgun_launcher/ClassLoaderCache.java b/nailgun_launcher/ClassLoaderCache.java new file mode 100644 index 0000000..6bffad0 --- /dev/null +++ b/nailgun_launcher/ClassLoaderCache.java @@ -0,0 +1,66 @@ +package cbt; + +import java.util.*; +import static java.io.File.pathSeparator; +import static cbt.Stage0Lib.*; + +final public class ClassLoaderCache{ + public Map hashMap; + final ThreadLocal> seen = new ThreadLocal>(){ + @Override protected HashSet initialValue(){ + return new HashSet(); + } + }; + + public ClassLoaderCache( + Map hashMap + ){ + this.hashMap = hashMap; + } + + public ClassLoader get( String key, long timestamp ){ + seen.get().add( key ); + @SuppressWarnings("unchecked") + ClassLoader t = (ClassLoader) hashMap.get( + hashMap.get( key ) + ); + assert hashMap.get(t).equals(timestamp); + return t; + } + + public boolean containsKey( String key, long timestamp ){ + boolean contains = hashMap.containsKey( key ); + if( contains ){ + Object keyObject = hashMap.get( key ); + Object classLoader = hashMap.get( keyObject ); + long oldTimestamp = (long) hashMap.get( classLoader ); + boolean res = oldTimestamp == timestamp; + return res; + } else { + return false; + } + } + + public void put( String key, ClassLoader value, long timestamp ){ + assert !seen.get().contains( key ): "Thread tries to update cache key after observing it: " + key; + LockableJavaKey keyObject = new LockableJavaKey(); + hashMap.put( key, keyObject ); + hashMap.put( keyObject, value ); + hashMap.put( value, timestamp ); + } + + @Override public String toString(){ + StringBuilder res = new StringBuilder(); + res.append("ClassLoaderCache(\n\n"); + for( Object key: hashMap.keySet() ){ + if( key instanceof String ) + res.append( + join( "\n", key.toString().split(":") ) + " -> " + hashMap.get( hashMap.get(key) ) + + "\n\n" + ); + } + res.append("\n\n"); + return res.toString(); + } +} +class LockableJavaKey{} diff --git a/nailgun_launcher/EarlyDependencies.java b/nailgun_launcher/EarlyDependencies.java index fdb54b5..8709e69 100644 --- a/nailgun_launcher/EarlyDependencies.java +++ b/nailgun_launcher/EarlyDependencies.java @@ -4,6 +4,7 @@ import java.io.*; import java.nio.file.*; import java.net.*; import java.security.*; +import java.util.*; import static cbt.Stage0Lib.*; import static cbt.NailgunLauncher.*; @@ -28,7 +29,7 @@ class EarlyDependencies{ String scalaLibrary_2_10_6_File; public EarlyDependencies( - String mavenCache, String mavenUrl, JavaCache classLoaderCache, ClassLoader rootClassLoader + String mavenCache, String mavenUrl, ClassLoaderCache classLoaderCache, ClassLoader rootClassLoader ) throws Throwable { scalaReflect_2_11_8_File = mavenCache + "/org/scala-lang/scala-reflect/2.11.8/scala-reflect-2.11.8.jar"; scalaCompiler_2_11_8_File = mavenCache + "/org/scala-lang/scala-compiler/2.11.8/scala-compiler-2.11.8.jar"; @@ -46,95 +47,104 @@ class EarlyDependencies{ download(new URL(mavenUrl + "/org/scala-lang/scala-compiler/2.11.8/scala-compiler-2.11.8.jar"), Paths.get(scalaCompiler_2_11_8_File), "fe1285c9f7b58954c5ef6d80b59063569c065e9a"); // org.scala-lang:scala-library:2.10.6 - download(new URL(mavenUrl + "/org/scala-lang/scala-library/2.10.6/scala-library-2.10.6.jar"), Paths.get(scalaLibrary_2_10_6_File), "421989aa8f95a05a4f894630aad96b8c7b828732"); - String[] scalaLibrary_2_10_6_ClasspathArray = new String[]{scalaLibrary_2_10_6_File}; - String scalaLibrary_2_10_6_Classpath = classpath( scalaLibrary_2_10_6_ClasspathArray ); - ClassLoader scalaLibrary_2_10_6_ = - classLoaderCache.contains( scalaLibrary_2_10_6_Classpath ) - ? classLoaderCache.get( scalaLibrary_2_10_6_Classpath ) - : classLoaderCache.put( classLoader( scalaLibrary_2_10_6_File, rootClassLoader ), scalaLibrary_2_10_6_Classpath ); + ClassLoader scalaLibrary_2_10_6_ = loadDependency( + mavenUrl + "/org/scala-lang/scala-library/2.10.6/scala-library-2.10.6.jar", + scalaLibrary_2_10_6_File, + "421989aa8f95a05a4f894630aad96b8c7b828732", + classLoaderCache, + rootClassLoader, + scalaLibrary_2_10_6_ClasspathArray + ); // org.scala-lang:scala-reflect:2.10.6 - download(new URL(mavenUrl + "/org/scala-lang/scala-reflect/2.10.6/scala-reflect-2.10.6.jar"), Paths.get(scalaReflect_2_10_6_File), "3259f3df0f166f017ef5b2d385445808398c316c"); - String[] scalaReflect_2_10_6_ClasspathArray = new String[]{scalaLibrary_2_10_6_File, scalaReflect_2_10_6_File}; - String scalaReflect_2_10_6_Classpath = classpath( scalaReflect_2_10_6_ClasspathArray ); - ClassLoader scalaReflect_2_10_6_ = - classLoaderCache.contains( scalaReflect_2_10_6_Classpath ) - ? classLoaderCache.get( scalaReflect_2_10_6_Classpath ) - : classLoaderCache.put( classLoader( scalaReflect_2_10_6_File, scalaLibrary_2_10_6_ ), scalaReflect_2_10_6_Classpath ); + ClassLoader scalaReflect_2_10_6_ = loadDependency( + mavenUrl + "/org/scala-lang/scala-reflect/2.10.6/scala-reflect-2.10.6.jar", + scalaReflect_2_10_6_File, + "3259f3df0f166f017ef5b2d385445808398c316c", + classLoaderCache, + scalaLibrary_2_10_6_, + scalaReflect_2_10_6_ClasspathArray + ); // com.typesafe.sbt:sbt-interface:0.13.12 - download(new URL(mavenUrl + "/com/typesafe/sbt/sbt-interface/0.13.12/sbt-interface-0.13.12.jar"), Paths.get(sbtInterface_0_13_12_File), "fcc7875c02f0d4641fac0518121bd71475d3909b"); - String[] sbtInterface_0_13_12_ClasspathArray = new String[]{sbtInterface_0_13_12_File, scalaLibrary_2_10_6_File, scalaReflect_2_10_6_File}; - String sbtInterface_0_13_12_Classpath = classpath( sbtInterface_0_13_12_ClasspathArray ); - ClassLoader sbtInterface_0_13_12_ = - classLoaderCache.contains( sbtInterface_0_13_12_Classpath ) - ? classLoaderCache.get( sbtInterface_0_13_12_Classpath ) - : classLoaderCache.put( classLoader( sbtInterface_0_13_12_File, scalaReflect_2_10_6_ ), sbtInterface_0_13_12_Classpath ); + ClassLoader sbtInterface_0_13_12_ = loadDependency( + mavenUrl + "/com/typesafe/sbt/sbt-interface/0.13.12/sbt-interface-0.13.12.jar", + sbtInterface_0_13_12_File, + "fcc7875c02f0d4641fac0518121bd71475d3909b", + classLoaderCache, + scalaReflect_2_10_6_, + sbtInterface_0_13_12_ClasspathArray + ); // org.scala-lang:scala-compiler:2.10.6 - download(new URL(mavenUrl + "/org/scala-lang/scala-compiler/2.10.6/scala-compiler-2.10.6.jar"), Paths.get(scalaCompiler_2_10_6_File), "9b15174852f5b6bb1edbf303d5722286a0a54011"); - String[] scalaCompiler_2_10_6_ClasspathArray = new String[]{sbtInterface_0_13_12_File, scalaCompiler_2_10_6_File, scalaLibrary_2_10_6_File, scalaReflect_2_10_6_File}; - String scalaCompiler_2_10_6_Classpath = classpath( scalaCompiler_2_10_6_ClasspathArray ); - ClassLoader scalaCompiler_2_10_6_ = - classLoaderCache.contains( scalaCompiler_2_10_6_Classpath ) - ? classLoaderCache.get( scalaCompiler_2_10_6_Classpath ) - : classLoaderCache.put( classLoader( scalaCompiler_2_10_6_File, sbtInterface_0_13_12_ ), scalaCompiler_2_10_6_Classpath ); + ClassLoader scalaCompiler_2_10_6_ = loadDependency( + mavenUrl + "/org/scala-lang/scala-compiler/2.10.6/scala-compiler-2.10.6.jar", + scalaCompiler_2_10_6_File, + "9b15174852f5b6bb1edbf303d5722286a0a54011", + classLoaderCache, + sbtInterface_0_13_12_, + scalaCompiler_2_10_6_ClasspathArray + ); // com.typesafe.sbt:compiler-interface:0.13.12 - download(new URL(mavenUrl + "/com/typesafe/sbt/compiler-interface/0.13.12/compiler-interface-0.13.12-sources.jar"), Paths.get(compilerInterface_0_13_12_File), "d9c3270576e162bf017b146af262364c2db87a32"); - String[] compilerInterface_0_13_12_ClasspathArray = new String[]{compilerInterface_0_13_12_File, sbtInterface_0_13_12_File, scalaCompiler_2_10_6_File, scalaLibrary_2_10_6_File, scalaReflect_2_10_6_File}; - String compilerInterface_0_13_12_Classpath = classpath( compilerInterface_0_13_12_ClasspathArray ); - ClassLoader compilerInterface_0_13_12_ = - classLoaderCache.contains( compilerInterface_0_13_12_Classpath ) - ? classLoaderCache.get( compilerInterface_0_13_12_Classpath ) - : classLoaderCache.put( classLoader( compilerInterface_0_13_12_File, scalaCompiler_2_10_6_ ), compilerInterface_0_13_12_Classpath ); + ClassLoader compilerInterface_0_13_12_ = loadDependency( + mavenUrl + "/com/typesafe/sbt/compiler-interface/0.13.12/compiler-interface-0.13.12-sources.jar", + compilerInterface_0_13_12_File, + "d9c3270576e162bf017b146af262364c2db87a32", + classLoaderCache, + scalaCompiler_2_10_6_, + compilerInterface_0_13_12_ClasspathArray + ); // com.typesafe.sbt:incremental-compiler:0.13.12 - download(new URL(mavenUrl + "/com/typesafe/sbt/incremental-compiler/0.13.12/incremental-compiler-0.13.12.jar"), Paths.get(incrementalCompiler_0_13_12_File), "259f6d24a5a3791bb233787d6a8e639c4ab86fe5"); - String[] incrementalCompiler_0_13_12_ClasspathArray = new String[]{compilerInterface_0_13_12_File, incrementalCompiler_0_13_12_File, sbtInterface_0_13_12_File, scalaCompiler_2_10_6_File, scalaLibrary_2_10_6_File, scalaReflect_2_10_6_File}; - String incrementalCompiler_0_13_12_Classpath = classpath( incrementalCompiler_0_13_12_ClasspathArray ); - ClassLoader incrementalCompiler_0_13_12_ = - classLoaderCache.contains( incrementalCompiler_0_13_12_Classpath ) - ? classLoaderCache.get( incrementalCompiler_0_13_12_Classpath ) - : classLoaderCache.put( classLoader( incrementalCompiler_0_13_12_File, compilerInterface_0_13_12_ ), incrementalCompiler_0_13_12_Classpath ); + ClassLoader incrementalCompiler_0_13_12_ = loadDependency( + mavenUrl + "/com/typesafe/sbt/incremental-compiler/0.13.12/incremental-compiler-0.13.12.jar", + incrementalCompiler_0_13_12_File, + "259f6d24a5a3791bb233787d6a8e639c4ab86fe5", + classLoaderCache, + compilerInterface_0_13_12_, + incrementalCompiler_0_13_12_ClasspathArray + ); // com.typesafe.zinc:zinc:0.3.12 - download(new URL(mavenUrl + "/com/typesafe/zinc/zinc/0.3.12/zinc-0.3.12.jar"), Paths.get(zinc_0_3_12_File), "c4339e93f5b7273f49ad026248f4fdb1d4d6c7c4"); - String[] zinc_0_3_12_ClasspathArray = new String[]{compilerInterface_0_13_12_File, incrementalCompiler_0_13_12_File, sbtInterface_0_13_12_File, zinc_0_3_12_File, scalaCompiler_2_10_6_File, scalaLibrary_2_10_6_File, scalaReflect_2_10_6_File}; - String zinc_0_3_12_Classpath = classpath( zinc_0_3_12_ClasspathArray ); - ClassLoader zinc_0_3_12_ = - classLoaderCache.contains( zinc_0_3_12_Classpath ) - ? classLoaderCache.get( zinc_0_3_12_Classpath ) - : classLoaderCache.put( classLoader( zinc_0_3_12_File, incrementalCompiler_0_13_12_ ), zinc_0_3_12_Classpath ); + ClassLoader zinc_0_3_12_ = loadDependency( + mavenUrl + "/com/typesafe/zinc/zinc/0.3.12/zinc-0.3.12.jar", + zinc_0_3_12_File, + "c4339e93f5b7273f49ad026248f4fdb1d4d6c7c4", + classLoaderCache, + incrementalCompiler_0_13_12_, + zinc_0_3_12_ClasspathArray + ); // org.scala-lang:scala-library:2.11.8 - download(new URL(mavenUrl + "/org/scala-lang/scala-library/2.11.8/scala-library-2.11.8.jar"), Paths.get(scalaLibrary_2_11_8_File), "ddd5a8bced249bedd86fb4578a39b9fb71480573"); - String[] scalaLibrary_2_11_8_ClasspathArray = new String[]{scalaLibrary_2_11_8_File}; - String scalaLibrary_2_11_8_Classpath = classpath( scalaLibrary_2_11_8_ClasspathArray ); - ClassLoader scalaLibrary_2_11_8_ = - classLoaderCache.contains( scalaLibrary_2_11_8_Classpath ) - ? classLoaderCache.get( scalaLibrary_2_11_8_Classpath ) - : classLoaderCache.put( classLoader( scalaLibrary_2_11_8_File, rootClassLoader ), scalaLibrary_2_11_8_Classpath ); + ClassLoader scalaLibrary_2_11_8_ = loadDependency( + mavenUrl + "/org/scala-lang/scala-library/2.11.8/scala-library-2.11.8.jar", + scalaLibrary_2_11_8_File, + "ddd5a8bced249bedd86fb4578a39b9fb71480573", + classLoaderCache, + rootClassLoader, + scalaLibrary_2_11_8_ClasspathArray + ); // org.scala-lang.modules:scala-xml_2.11:1.0.5 - download(new URL(mavenUrl + "/org/scala-lang/modules/scala-xml_2.11/1.0.5/scala-xml_2.11-1.0.5.jar"), Paths.get(scalaXml_1_0_5_File), "77ac9be4033768cf03cc04fbd1fc5e5711de2459"); - String[] scalaXml_1_0_5_ClasspathArray = new String[]{scalaXml_1_0_5_File, scalaLibrary_2_11_8_File}; - String scalaXml_1_0_5_Classpath = classpath( scalaXml_1_0_5_ClasspathArray ); - ClassLoader scalaXml_1_0_5_ = - classLoaderCache.contains( scalaXml_1_0_5_Classpath ) - ? classLoaderCache.get( scalaXml_1_0_5_Classpath ) - : classLoaderCache.put( classLoader( scalaXml_1_0_5_File, scalaLibrary_2_11_8_ ), scalaXml_1_0_5_Classpath ); - + ClassLoader scalaXml_1_0_5_ = loadDependency( + mavenUrl + "/org/scala-lang/modules/scala-xml_2.11/1.0.5/scala-xml_2.11-1.0.5.jar", + scalaXml_1_0_5_File, + "77ac9be4033768cf03cc04fbd1fc5e5711de2459", + classLoaderCache, + scalaLibrary_2_11_8_, + scalaXml_1_0_5_ClasspathArray + ); + classLoader = scalaXml_1_0_5_; classpathArray = scalaXml_1_0_5_ClasspathArray; diff --git a/nailgun_launcher/JavaCache.java b/nailgun_launcher/JavaCache.java deleted file mode 100644 index 3ba12ab..0000000 --- a/nailgun_launcher/JavaCache.java +++ /dev/null @@ -1,40 +0,0 @@ -package cbt; - -import java.util.*; -import static java.io.File.pathSeparator; -import static cbt.Stage0Lib.*; - -final class JavaCache{ - Map hashMap; - - public JavaCache( - Map hashMap - ){ - this.hashMap = hashMap; - } - - public T get( Object key ){ - @SuppressWarnings("unchecked") - T t = (T) hashMap.get( - hashMap.get( key ) - ); - return t; - } - - public Boolean contains( Object key/*, Long timestamp*/ ){ - return hashMap.containsKey( key );/* && ( - (Long) hashMap.get( hashMap.get( hashMap.get(key) ) ) >= timestamp - );*/ - } - - public T put( Object value, Object key/*, Long timestamp*/ ){ - LockableJavaKey keyObject = new LockableJavaKey(); - hashMap.put( key, keyObject ); - hashMap.put( keyObject, value ); - //hashMap.put( value, timestamp ); - @SuppressWarnings("unchecked") - T t = (T) value; - return t; - } -} -class LockableJavaKey{} diff --git a/nailgun_launcher/NailgunLauncher.java b/nailgun_launcher/NailgunLauncher.java index c397810..8a330d8 100644 --- a/nailgun_launcher/NailgunLauncher.java +++ b/nailgun_launcher/NailgunLauncher.java @@ -26,12 +26,12 @@ public class NailgunLauncher{ @SuppressWarnings("unchecked") public static Object getBuild( Object context ) throws Throwable{ BuildStage1Result res = buildStage1( - (Boolean) get(context, "cbtHasChangedCompat"), - (Long) get(context, "startCompat"), + (long) get(context, "cbtLastModified"), + (long) get(context, "start"), ((File) get(context, "cache")).toString() + "/", ((File) get(context, "cbtHome")).toString(), ((File) get(context, "compatibilityTarget")).toString() + "/", - new JavaCache( + new ClassLoaderCache( (HashMap) get(context, "persistentCache") ) ); @@ -43,6 +43,8 @@ public class NailgunLauncher{ .invoke(null, context, res); } + public static long nailgunLauncherLastModified = -1; // this initial value should be overwritten, never read + public static void main( String[] args ) throws Throwable { Long _start = System.currentTimeMillis(); if(args[0].equals("check-alive")){ @@ -76,13 +78,18 @@ public class NailgunLauncher{ String CBT_HOME = System.getenv("CBT_HOME"); String cache = CBT_HOME + "/cache/"; String compatibilityTarget = CBT_HOME + "/compatibility/" + TARGET; - // copy cache, so that this thread has a consistent view - // replace before returning, see below - JavaCache classLoaderCache = new JavaCache( + // copy cache, so that this thread has a consistent view despite other threads + // changing their copies + // replace again before returning, see below + ClassLoaderCache classLoaderCache = new ClassLoaderCache( new HashMap(classLoaderCacheHashMap) ); + + String nailgunTarget = CBT_HOME + "/" + NAILGUN + TARGET; + long nailgunLauncherLastModified = new File( nailgunTarget + "../classes.last-success" ).lastModified(); + BuildStage1Result res = buildStage1( - false, start, cache, CBT_HOME, compatibilityTarget, classLoaderCache + nailgunLauncherLastModified, start, cache, CBT_HOME, compatibilityTarget, classLoaderCache ); try{ @@ -90,9 +97,9 @@ public class NailgunLauncher{ .classLoader .loadClass("cbt.Stage1") .getMethod( - "run", String[].class, File.class, File.class, BuildStage1Result.class, Long.class, Map.class + "run", String[].class, File.class, File.class, BuildStage1Result.class, Map.class ).invoke( - null, (Object) args, new File(cache), new File(CBT_HOME), res, start, classLoaderCache.hashMap + null, (Object) args, new File(cache), new File(CBT_HOME), res, classLoaderCache.hashMap ); System.exit( exitCode ); @@ -115,7 +122,8 @@ public class NailgunLauncher{ } public static BuildStage1Result buildStage1( - Boolean changed, long start, String cache, String cbtHome, String compatibilityTarget, JavaCache classLoaderCache + final long lastModified, final long start, final String cache, final String cbtHome, + final String compatibilityTarget, final ClassLoaderCache classLoaderCache ) throws Throwable { _assert(TARGET != null, "environment variable TARGET not defined"); String nailgunTarget = cbtHome + "/" + NAILGUN + TARGET; @@ -128,9 +136,11 @@ public class NailgunLauncher{ ClassLoader rootClassLoader = new CbtURLClassLoader( new URL[]{}, ClassLoader.getSystemClassLoader().getParent() ); // wrap for caching EarlyDependencies earlyDeps = new EarlyDependencies(mavenCache, mavenUrl, classLoaderCache, rootClassLoader); - ClassLoader compatibilityClassLoader; + long nailgunLauncherLastModified = new File( nailgunTarget + "../classes.last-success" ).lastModified(); + + long compatibilityLastModified; if(!compatibilityTarget.startsWith(cbtHome)){ - compatibilityClassLoader = classLoaderCache.get( compatibilityTarget ); + compatibilityLastModified = new File( compatibilityTarget + "../classes.last-success" ).lastModified(); } else { List compatibilitySourceFiles = new ArrayList(); for( File f: compatibilitySources.listFiles() ){ @@ -138,20 +148,20 @@ public class NailgunLauncher{ compatibilitySourceFiles.add(f); } } - changed = compile(changed, start, "", compatibilityTarget, earlyDeps, compatibilitySourceFiles); - - if( classLoaderCache.contains( compatibilityTarget ) ){ - compatibilityClassLoader = classLoaderCache.get( compatibilityTarget ); - } else { - compatibilityClassLoader = classLoaderCache.put( classLoader(compatibilityTarget, rootClassLoader), compatibilityTarget ); + + compatibilityLastModified = compile( 0L, "", compatibilityTarget, earlyDeps, compatibilitySourceFiles); + + if( !classLoaderCache.containsKey( compatibilityTarget, compatibilityLastModified ) ){ + classLoaderCache.put( compatibilityTarget, classLoader(compatibilityTarget, rootClassLoader), compatibilityLastModified ); } } + final ClassLoader compatibilityClassLoader = classLoaderCache.get( compatibilityTarget, compatibilityLastModified ); String[] nailgunClasspathArray = append( earlyDeps.classpathArray, nailgunTarget ); String nailgunClasspath = classpath( nailgunClasspathArray ); - ClassLoader nailgunClassLoader = new CbtURLClassLoader( new URL[]{}, NailgunLauncher.class.getClassLoader() ); // wrap for caching - if( !classLoaderCache.contains( nailgunClasspath ) ){ - nailgunClassLoader = classLoaderCache.put( nailgunClassLoader, nailgunClasspath ); + final ClassLoader nailgunClassLoader = new CbtURLClassLoader( new URL[]{}, NailgunLauncher.class.getClassLoader() ); // wrap for caching + if( !classLoaderCache.containsKey( nailgunClasspath, nailgunLauncherLastModified ) ){ + classLoaderCache.put( nailgunClasspath, nailgunClassLoader, nailgunLauncherLastModified ); } String[] stage1ClasspathArray = @@ -164,14 +174,24 @@ public class NailgunLauncher{ stage1SourceFiles.add(f); } } - changed = compile(changed, start, stage1Classpath, stage1Target, earlyDeps, stage1SourceFiles); - ClassLoader stage1classLoader; - if( !changed && classLoaderCache.contains( stage1Classpath ) ){ - stage1classLoader = classLoaderCache.get( stage1Classpath ); - } else { - stage1classLoader = + final long stage1BeforeCompiled = System.currentTimeMillis(); + final long stage0LastModified = Math.max( + lastModified, + Math.max( lastModified, compatibilityLastModified ) + ); + final long stage1LastModified = compile( + stage0LastModified, stage1Classpath, stage1Target, earlyDeps, stage1SourceFiles + ); + + if( stage1LastModified < compatibilityLastModified ) + throw new AssertionError( + "Cache invalidation bug: cbt compatibility layer recompiled, but cbt stage1 did not." + ); + + if( !classLoaderCache.containsKey( stage1Classpath, stage1LastModified ) ){ classLoaderCache.put( + stage1Classpath, classLoader( stage1Target, new MultiClassLoader2( @@ -180,12 +200,14 @@ public class NailgunLauncher{ earlyDeps.classLoader ) ), - stage1Classpath + stage1LastModified ); } + final ClassLoader stage1classLoader = classLoaderCache.get( stage1Classpath, stage1LastModified ); return new BuildStage1Result( - changed, + start, + stage1LastModified, stage1classLoader, stage1Classpath, nailgunClasspath, diff --git a/nailgun_launcher/Stage0Lib.java b/nailgun_launcher/Stage0Lib.java index 865b1cb..425ced3 100644 --- a/nailgun_launcher/Stage0Lib.java +++ b/nailgun_launcher/Stage0Lib.java @@ -11,6 +11,7 @@ import static java.io.File.pathSeparator; import static cbt.NailgunLauncher.*; import java.nio.file.*; import java.nio.file.attribute.FileTime; +import static java.lang.Math.min; public class Stage0Lib{ public static void _assert(Boolean condition, Object msg){ @@ -47,25 +48,57 @@ public class Stage0Lib{ return join( pathSeparator, files ); } + public static long lastModified( String... files ){ + List lastModified = new ArrayList(); + for( String file: files ){ + lastModified.add( new File(file).lastModified() ); + } + return Collections.max( lastModified ); + } + + public static ClassLoader loadDependency( + String url, + String file, + String hash, + ClassLoaderCache classLoaderCache, + ClassLoader parent, + String... classpathArray + ) throws Throwable { + download(new URL(url), Paths.get(file), hash); + + final long lastModified = lastModified( classpathArray ); + final String classpath = classpath( classpathArray ); + + if( !classLoaderCache.containsKey( classpath, lastModified ) ) + classLoaderCache.put( classpath, classLoader( file, parent ), lastModified ); + + return classLoaderCache.get( classpath, lastModified ); + } + public static File write(File file, String content, OpenOption... options) throws Throwable{ file.getParentFile().mkdirs(); Files.write(file.toPath(), content.getBytes(), options); return file; } - public static Boolean compile( - Boolean changed, Long start, String classpath, String target, + public static long compile( + long lastModified, String classpath, String target, EarlyDependencies earlyDeps, List sourceFiles ) throws Throwable{ File statusFile = new File( new File(target) + ".last-success" ); - Long lastSuccessfullCompile = statusFile.lastModified(); + long lastCompiled = statusFile.lastModified(); + + long maxLastModified = lastModified; + final long start = System.currentTimeMillis(); // <- before recursing, so we catch them all + for( File file: sourceFiles ){ - if( file.lastModified() > lastSuccessfullCompile ){ - changed = true; - break; - } + long l = file.lastModified(); + if( l > maxLastModified ) maxLastModified = l; + // performance optimization because we'll recompile and don't need to check other files + if( l > lastCompiled ) break; } - if(changed){ + + if( maxLastModified > lastCompiled ){ List zincArgs = new ArrayList( Arrays.asList( new String[]{ @@ -100,8 +133,10 @@ public class Stage0Lib{ } finally { System.setOut(oldOut); } + return statusFile.lastModified(); // can't just use `start` here as system time precision is less than milliseconds on OSX + } else { + return lastCompiled; } - return changed; } public static ClassLoader classLoader( String file ) throws Throwable{ diff --git a/plugins/scalajs/ScalaJsBuild.scala b/plugins/scalajs/ScalaJsBuild.scala index 9374f66..99f8616 100644 --- a/plugins/scalajs/ScalaJsBuild.scala +++ b/plugins/scalajs/ScalaJsBuild.scala @@ -6,7 +6,7 @@ trait ScalaJsBuild extends BaseBuild { final protected val scalaJsLib = ScalaJsLib( scalaJsVersion, scalaVersion, - context.cbtHasChanged, + context.cbtLastModified, context.classLoaderCache, context.paths.mavenCache ) diff --git a/plugins/scalajs/ScalaJsLib.scala b/plugins/scalajs/ScalaJsLib.scala index 12c1c85..f500039 100644 --- a/plugins/scalajs/ScalaJsLib.scala +++ b/plugins/scalajs/ScalaJsLib.scala @@ -3,7 +3,7 @@ import java.io.File case class ScalaJsLib( scalaJsVersion: String, scalaVersion: String, - cbtHasChanged: Boolean, classLoaderCache: ClassLoaderCache, mavenCache: File + cbtLastModified: Long, classLoaderCache: ClassLoaderCache, mavenCache: File )(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]){ sealed trait ScalaJsOutputMode { def option: String @@ -19,7 +19,7 @@ case class ScalaJsLib( } val lib = new Lib(logger) - def dep(artifactId: String) = MavenResolver( cbtHasChanged, mavenCache, mavenCentral ).bindOne( + def dep(artifactId: String) = MavenResolver( cbtLastModified, mavenCache, mavenCentral ).bindOne( MavenDependency("org.scala-js", artifactId, scalaJsVersion) ) diff --git a/plugins/scalatest/ScalaTest.scala b/plugins/scalatest/ScalaTest.scala index ee96431..5ccabc6 100644 --- a/plugins/scalatest/ScalaTest.scala +++ b/plugins/scalatest/ScalaTest.scala @@ -6,7 +6,7 @@ trait ScalaTest extends BaseBuild{ override def run: ExitCode = { import ScalaTestLib._ val _classLoader = classLoader(context.classLoaderCache) - val suiteNames = compile.map( d => discoverSuites(d, _classLoader) ).toVector.flatten + val suiteNames = compileFile.map( d => discoverSuites(d, _classLoader) ).toVector.flatten runSuites( suiteNames.map( loadSuite( _, _classLoader ) ) ) ExitCode.Success } diff --git a/plugins/wartremover/WartRemover.scala b/plugins/wartremover/WartRemover.scala index d5bbcd0..9cf8851 100644 --- a/plugins/wartremover/WartRemover.scala +++ b/plugins/wartremover/WartRemover.scala @@ -10,7 +10,7 @@ trait WartRemover extends BaseBuild { private[this] def wartremoverCompilerDependency: String = MavenResolver( - context.cbtHasChanged, + context.cbtLastModified, context.paths.mavenCache, mavenCentral).bindOne( ScalaDependency("org.wartremover", "wartremover", "1.1.1") diff --git a/stage1/CbtPaths.scala b/stage1/CbtPaths.scala index 71c2ef1..c8f2279 100644 --- a/stage1/CbtPaths.scala +++ b/stage1/CbtPaths.scala @@ -9,7 +9,9 @@ case class CbtPaths(private val cbtHome: File, private val cache: File){ private val target = NailgunLauncher.TARGET.stripSuffix("/") val stage1Target: File = stage1 ++ ("/" ++ target) val stage2Target: File = stage2 ++ ("/" ++ target) + val stage1StatusFile: File = stage1Target ++ ".last-success" val stage2StatusFile: File = stage2Target ++ ".last-success" val compatibility: File = cbtHome ++ "/compatibility" val nailgunTarget: File = nailgun ++ ("/" ++ target) + val nailgunStatusFile: File = nailgunTarget ++ ".last-success" } diff --git a/stage1/ClassLoaderCache.scala b/stage1/ClassLoaderCache.scala deleted file mode 100644 index af0970e..0000000 --- a/stage1/ClassLoaderCache.scala +++ /dev/null @@ -1,24 +0,0 @@ -package cbt - -import java.net._ -import java.util._ -import collection.JavaConverters._ - -case class ClassLoaderCache( - logger: Logger, - private[cbt] hashMap: java.util.Map[AnyRef,AnyRef] -){ - val cache = new KeyLockedLazyCache[ClassLoader]( hashMap, Some(logger) ) - override def toString = ( - s"ClassLoaderCache(" - ++ - hashMap.asScala.collect{ - case (key, value) if key.isInstanceOf[String] => - key.toString.split(":").mkString("\n") -> value - }.toVector.sortBy(_._1).map{ - case (key, value) => key + " -> " + hashMap.get(value) - }.mkString("\n\n","\n\n","\n\n") - ++ - ")" - ) -} diff --git a/stage1/ContextImplementation.scala b/stage1/ContextImplementation.scala index 30db597..69094b0 100644 --- a/stage1/ContextImplementation.scala +++ b/stage1/ContextImplementation.scala @@ -1,20 +1,28 @@ package cbt import java.io._ -import java.lang._ class ContextImplementation( - val projectDirectory: File, - val cwd: File, - val argsArray: Array[String], - val enabledLoggersArray: Array[String], - val startCompat: Long, - val cbtHasChangedCompat: Boolean, - val scalaVersionOrNull: String, - val persistentCache: java.util.Map[AnyRef,AnyRef], - val transientCache: java.util.Map[AnyRef,AnyRef], - val cache: File, - val cbtHome: File, - val cbtRootHome: File, - val compatibilityTarget: File, - val parentBuildOrNull: BuildInterface -) extends Context + override val projectDirectory: File, + override val cwd: File, + override val argsArray: Array[String], + override val enabledLoggersArray: Array[String], + override val start: Long, + override val cbtLastModified: Long, + override val scalaVersionOrNull: String, + override val persistentCache: java.util.Map[AnyRef,AnyRef], + override val transientCache: java.util.Map[AnyRef,AnyRef], + override val cache: File, + override val cbtHome: File, + override val cbtRootHome: File, + override val compatibilityTarget: File, + override val parentBuildOrNull: BuildInterface +) extends Context{ + @deprecated("this method is replaced by cbtLastModified","") + def cbtHasChangedCompat = true + @deprecated("this method is replaced by start","") + def startCompat = start + @deprecated("this methods is replaced by persistentCache","") + def permanentKeys = throw new IncompatibleCbtVersionException("You need to upgrade your CBT version in this module. The Context field permanentClassLoaders is no longer supported."); + @deprecated("this methods is replaced by persistentCache","") + def permanentClassLoaders = throw new IncompatibleCbtVersionException("You need to upgrade your CBT version in this module. The Context field permanentClassLoaders is no longer supported."); +} diff --git a/stage1/MavenRepository.scala b/stage1/MavenRepository.scala index 2ac7064..a8c9b51 100644 --- a/stage1/MavenRepository.scala +++ b/stage1/MavenRepository.scala @@ -2,12 +2,12 @@ package cbt import java.io._ import java.net._ case class MavenResolver( - cbtHasChanged: Boolean, mavenCache: File, urls: URL* + cbtLastModified: Long, mavenCache: File, urls: URL* )( implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef] ){ def bind( dependencies: MavenDependency* ): Seq[BoundMavenDependency] - = dependencies.map( BoundMavenDependency(cbtHasChanged,mavenCache,_,urls.to) ).to + = dependencies.map( BoundMavenDependency(cbtLastModified,mavenCache,_,urls.to) ).to def bindOne( dependency: MavenDependency ): BoundMavenDependency - = BoundMavenDependency( cbtHasChanged, mavenCache, dependency, urls.to ) + = BoundMavenDependency( cbtLastModified, mavenCache, dependency, urls.to ) } diff --git a/stage1/Stage1.scala b/stage1/Stage1.scala index 27c6402..1fd4663 100644 --- a/stage1/Stage1.scala +++ b/stage1/Stage1.scala @@ -39,7 +39,8 @@ abstract class Stage2Base{ case class Stage2Args( cwd: File, args: Seq[String], - cbtHasChanged: Boolean, + stage2LastModified: Long, + logger: Logger, classLoaderCache: ClassLoaderCache, cache: File, cbtHome: File, @@ -47,8 +48,9 @@ case class Stage2Args( )( implicit val transientCache: java.util.Map[AnyRef,AnyRef] ){ - val ClassLoaderCache( logger, persistentCache ) = classLoaderCache + val persistentCache = classLoaderCache.hashMap } + object Stage1{ protected def newerThan( a: File, b: File ) ={ a.lastModified > b.lastModified @@ -56,13 +58,14 @@ object Stage1{ def getBuild( _context: java.lang.Object, buildStage1: BuildStage1Result ) = { val context = _context.asInstanceOf[Context] - val logger = new Logger( context.enabledLoggers, context.start ) - val (changed, classLoader) = buildStage2( + val logger = new Logger( context.enabledLoggers, buildStage1.start ) + val (cbtLastModified, classLoader) = buildStage2( buildStage1, - ClassLoaderCache( logger, context.persistentCache ), + new ClassLoaderCache( context.persistentCache ), context.cbtHome, - context.cache - )( context.transientCache ) + context.cache, + logger + )(context.transientCache) classLoader .loadClass("cbt.Stage2") @@ -70,15 +73,16 @@ object Stage1{ .invoke( null, context.copy( - cbtHasChanged = context.cbtHasChanged || buildStage1.changed || changed // might be redundant + cbtLastModified = Math.max( context.cbtLastModified, cbtLastModified ) ) ) } def buildStage2( - buildStage1: BuildStage1Result, classLoaderCache: ClassLoaderCache, cbtHome: File, cache: File - )(implicit transientCache: java.util.Map[AnyRef,AnyRef]): (Boolean, ClassLoader) = { - import classLoaderCache.logger + buildStage1: BuildStage1Result, classLoaderCache: ClassLoaderCache, cbtHome: File, cache: File, logger: Logger + )(implicit transientCache: java.util.Map[AnyRef,AnyRef]): (Long, ClassLoader) = { + + import buildStage1._ val lib = new Stage1Lib(logger) import lib._ @@ -89,61 +93,58 @@ object Stage1{ stage2.listFiles ++ (stage2 ++ "/plugins").listFiles ).toVector.filter(_.isFile).filter(_.toString.endsWith(".scala")) - val cbtHasChanged = buildStage1.changed || lib.needsUpdate(stage2sourceFiles, stage2StatusFile) - val cls = this.getClass.getClassLoader.loadClass("cbt.NailgunLauncher") - val cbtDependency = new CbtDependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, stage2Target, new File(buildStage1.compatibilityClasspath)) + def cbtDependencies = new CbtDependencies( + mavenCache, nailgunTarget, stage1Target, stage2Target, + new File(buildStage1.compatibilityClasspath) + ) logger.stage1("Compiling stage2 if necessary") - compile( - cbtHasChanged, - cbtHasChanged, + val Some( stage2LastModified ) = compile( + buildStage1.stage1LastModified, stage2sourceFiles, stage2Target, stage2StatusFile, - cbtDependency.dependencies, + cbtDependencies.stage2Dependency.dependencies, mavenCache, Seq("-deprecation","-feature","-unchecked"), classLoaderCache, zincVersion = constants.zincVersion, scalaVersion = constants.scalaVersion )(transientCache) logger.stage1(s"calling CbtDependency.classLoader") - if( cbtHasChanged && classLoaderCache.cache.containsKey( cbtDependency.classpath.string ) ) { - classLoaderCache.cache.remove( cbtDependency.classpath.string ) - } else { - assert( - buildStage1.compatibilityClasspath === cbtDependency.stage1Dependency.compatibilityDependency.classpath.string, - "compatibility classpath different from NailgunLauncher" - ) - assert( - buildStage1.stage1Classpath === cbtDependency.stage1Dependency.classpath.string, - "stage1 classpath different from NailgunLauncher" - ) - assert( - classLoaderCache.cache.containsKey( cbtDependency.stage1Dependency.compatibilityDependency.classpath.string ), - "cbt unchanged, expected compatibility classloader to be cached" - ) - assert( - classLoaderCache.cache.containsKey( cbtDependency.stage1Dependency.classpath.string ), - "cbt unchanged, expected stage1/nailgun classloader to be cached" - ) - } - val stage2ClassLoader = cbtDependency.classLoader(classLoaderCache) + assert( + buildStage1.compatibilityClasspath === cbtDependencies.compatibilityDependency.classpath.string, + "compatibility classpath different from NailgunLauncher" + ) + assert( + buildStage1.stage1Classpath === cbtDependencies.stage1Dependency.classpath.string, + "stage1 classpath different from NailgunLauncher" + ) + assert( + classLoaderCache.containsKey( cbtDependencies.compatibilityDependency.classpath.string, cbtDependencies.compatibilityDependency.lastModified ), + "cbt unchanged, expected compatibility classloader to be cached" + ) + assert( + classLoaderCache.containsKey( cbtDependencies.stage1Dependency.classpath.string, cbtDependencies.stage1Dependency.lastModified ), + "cbt unchanged, expected stage1 classloader to be cached" + ) + + val stage2ClassLoader = cbtDependencies.stage2Dependency.classLoader(classLoaderCache) { // a few classloader sanity checks val compatibilityClassLoader = - cbtDependency.stage1Dependency.compatibilityDependency.classLoader(classLoaderCache) + cbtDependencies.compatibilityDependency.classLoader(classLoaderCache) assert( classOf[BuildInterface].getClassLoader == compatibilityClassLoader, classOf[BuildInterface].getClassLoader.toString ++ "\n\nis not the same as\n\n" ++ compatibilityClassLoader.toString ) //------------- val stage1ClassLoader = - cbtDependency.stage1Dependency.classLoader(classLoaderCache) + cbtDependencies.stage1Dependency.classLoader(classLoaderCache) assert( - classOf[Stage1Dependency].getClassLoader == stage1ClassLoader, - classOf[Stage1Dependency].getClassLoader.toString ++ "\n\nis not the same as\n\n" ++ stage1ClassLoader.toString + classOf[Stage1ArgsParser].getClassLoader == stage1ClassLoader, + classOf[Stage1ArgsParser].getClassLoader.toString ++ "\n\nis not the same as\n\n" ++ stage1ClassLoader.toString ) //------------- assert( @@ -152,7 +153,7 @@ object Stage1{ ) } - ( cbtHasChanged, stage2ClassLoader ) + ( stage2LastModified, stage2ClassLoader ) } def run( @@ -160,24 +161,23 @@ object Stage1{ cache: File, cbtHome: File, buildStage1: BuildStage1Result, - start: java.lang.Long, persistentCache: java.util.Map[AnyRef,AnyRef] ): Int = { val args = Stage1ArgsParser(_args.toVector) - val logger = new Logger(args.enabledLoggers, start) + val logger = new Logger(args.enabledLoggers, buildStage1.start) implicit val transientCache: java.util.Map[AnyRef,AnyRef] = new java.util.HashMap logger.stage1(s"Stage1 start") - val classLoaderCache = ClassLoaderCache( logger, persistentCache ) - + val classLoaderCache = new ClassLoaderCache( persistentCache ) - val (cbtHasChanged, classLoader) = buildStage2( buildStage1, classLoaderCache, cbtHome, cache ) + val (stage2LastModified, classLoader) = buildStage2( buildStage1, classLoaderCache, cbtHome, cache, logger ) val stage2Args = Stage2Args( new File( args.args(0) ), args.args.drop(1).dropWhile(_ == "direct").toVector, // launcher changes cause entire nailgun restart, so no need for them here - cbtHasChanged = cbtHasChanged, + stage2LastModified = stage2LastModified, + logger = logger, classLoaderCache = classLoaderCache, cache, cbtHome, diff --git a/stage1/Stage1Lib.scala b/stage1/Stage1Lib.scala index f0bb588..ad4b2d0 100644 --- a/stage1/Stage1Lib.scala +++ b/stage1/Stage1Lib.scala @@ -32,7 +32,7 @@ class BaseLib{ def realpath(name: File) = new File(java.nio.file.Paths.get(name.getAbsolutePath).normalize.toString) } -class Stage1Lib( val logger: Logger ) extends BaseLib{ +class Stage1Lib( logger: Logger ) extends BaseLib{ lib => implicit val implicitLogger: Logger = logger @@ -183,15 +183,10 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ } } - def needsUpdate( sourceFiles: Seq[File], statusFile: File ) = { - val lastCompile = statusFile.lastModified - sourceFiles.filter(_.lastModified > lastCompile).nonEmpty - } def compile( - cbtHasChanged: Boolean, - needsRecompile: Boolean, - files: Seq[File], + cbtLastModified: Long, + sourceFiles: Seq[File], compileTarget: File, statusFile: File, dependencies: Seq[Dependency], @@ -202,20 +197,23 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ scalaVersion: String )( implicit transientCache: java.util.Map[AnyRef, AnyRef] - ): Option[File] = { - val classpath = Dependencies(dependencies).classpath + ): Option[Long] = { + val d = Dependencies(dependencies) + val classpath = d.classpath val cp = classpath.string if(classpath.files.isEmpty) - throw new Exception("Trying to compile with empty classpath. Source files: " ++ files.toString) + throw new Exception("Trying to compile with empty classpath. Source files: " ++ sourceFiles.toString) - if( files.isEmpty ){ + if( sourceFiles.isEmpty ){ None }else{ - if( needsRecompile ){ - def Resolver(urls: URL*) = MavenResolver(cbtHasChanged, mavenCache, urls: _*) + val start = System.currentTimeMillis + val lastCompiled = statusFile.lastModified + if( d.lastModified > lastCompiled || sourceFiles.exists(_.lastModified > lastCompiled) ){ + def Resolver(urls: URL*) = MavenResolver(cbtLastModified, mavenCache, urls: _*) val zinc = Resolver(mavenCentral).bindOne(MavenDependency("com.typesafe.zinc","zinc", zincVersion)) val zincDeps = zinc.transitiveDependencies - + val sbtInterface = zincDeps .collect{ case d @ @@ -242,8 +240,6 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ val scalaReflect = Resolver(mavenCentral).bindOne(MavenDependency("org.scala-lang","scala-reflect",scalaVersion)).jar val scalaCompiler = Resolver(mavenCentral).bindOne(MavenDependency("org.scala-lang","scala-compiler",scalaVersion)).jar - val start = System.currentTimeMillis - val _class = "com.typesafe.zinc.Main" val dualArgs = Seq( @@ -264,7 +260,7 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ _class, dualArgs ++ singleArgs ++ Seq( "-cp", cp // let's put cp last. It so long - ) ++ files.map(_.toString), + ) ++ sourceFiles.map(_.toString), zinc.classLoader(classLoaderCache) ) } catch { @@ -283,7 +279,7 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ -cp \\ ${classpath.strings.mkString(":\\\n")} \\ \\ - ${files.sorted.mkString(" \\\n")} + ${sourceFiles.sorted.mkString(" \\\n")} """ ) @@ -300,8 +296,10 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ } else { System.exit(code.integer) // FIXME: let's find a better solution for error handling. Maybe a monad after all. } + Some( start ) + } else { + Some( lastCompiled ) } - Some( compileTarget ) } } def redirectOutToErr[T](code: => T): T = { @@ -360,11 +358,11 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ ) def cacheOnDisk[T] - ( cbtHasChanged: Boolean, cacheFile: File ) + ( cbtLastModified: Long, cacheFile: File ) ( deserialize: String => T ) ( serialize: T => String ) ( compute: => Seq[T] ) = { - if(!cbtHasChanged && cacheFile.exists){ + if(cacheFile.exists && cacheFile.lastModified > cbtLastModified ){ import collection.JavaConversions._ Files .readAllLines( cacheFile.toPath, StandardCharsets.UTF_8 ) @@ -380,7 +378,7 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ def dependencyTreeRecursion(root: Dependency, indent: Int = 0): String = ( ( " " * indent ) - ++ (if(root.needsUpdate) red(root.show) else root.show) + ++ root.show // (if(root.needsUpdate) red(root.show) else root.show) ++ root.dependencies.map( d => "\n" ++ dependencyTreeRecursion(d,indent + 1) ).mkString @@ -420,33 +418,40 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ def classLoaderRecursion( dependency: Dependency, latest: Map[(String,String),Dependency], cache: ClassLoaderCache ): ClassLoader = { // FIXME: shouldn't we be using KeyLockedLazyCache instead of hashmap directly here? - val d = dependency val dependencies = dependency.dependencies - def dependencyClassLoader( latest: Map[(String,String),Dependency], cache: ClassLoaderCache ): ClassLoader = { + val dependencyClassLoader: ClassLoader = { if( dependency.dependencies.isEmpty ){ // wrap for caching new cbt.URLClassLoader( ClassPath(), ClassLoader.getSystemClassLoader().getParent() ) } else if( dependencies.size == 1 ){ classLoaderRecursion( dependencies.head, latest, cache ) } else{ - val cp = d.dependencyClasspath.string - if( dependencies.exists(_.needsUpdate) && cache.cache.containsKey(cp) ){ - cache.cache.remove(cp) - } - def cl = new MultiClassLoader( dependencies.map( classLoaderRecursion(_, latest, cache) ) ) - if(d.isInstanceOf[BuildInterface]) + val lastModified = dependencies.map( _.lastModified ).max + val cp = dependency.dependencyClasspath.string + val cl = + new MultiClassLoader( + dependencies.map( classLoaderRecursion(_, latest, cache) ) + ) + if(dependency.isInstanceOf[BuildInterface]) cl // Don't cache builds right now. We need to fix invalidation first. - else - cache.cache.get( cp, cl ) + else{ + if( !cache.containsKey( cp, lastModified ) ){ + cache.put( cp, cl, lastModified ) + } + cache.get( cp, lastModified ) + } } } val a = actual( dependency, latest ) - def cl = new cbt.URLClassLoader( a.exportedClasspath, dependencyClassLoader(latest, cache) ) - if(d.isInstanceOf[BuildInterface]) - cl - else - cache.cache.get( a.classpath.string, cl ).asInstanceOf[ClassLoader] + def cl = new cbt.URLClassLoader( a.exportedClasspath, dependencyClassLoader ) + + val cp = a.classpath.string + val lastModified = a.lastModified + if( !cache.containsKey( cp, lastModified ) ){ + cache.put( cp, cl, lastModified ) + } + cache.get( cp, lastModified ) } } diff --git a/stage1/cbt.scala b/stage1/cbt.scala index 01c9303..0b0ccbf 100644 --- a/stage1/cbt.scala +++ b/stage1/cbt.scala @@ -33,6 +33,11 @@ object `package`{ def ++( s: String ): URL = new URL( url.toString ++ s ) def string = url.toString } + implicit class SeqExtensions[T](seq: Seq[T]){ + def maxOption(implicit ev: Ordering[T]): Option[T] = try{ Some(seq.max) } catch { + case e:java.lang.UnsupportedOperationException if e.getMessage === "empty.max" => None + } + } implicit class BuildInterfaceExtensions(build: BuildInterface){ import build._ // TODO: if every build has a method triggers a callback if files change @@ -52,30 +57,30 @@ object `package`{ def exportedClasspath: ClassPath = ClassPath(exportedClasspathArray.to) def classpath = exportedClasspath ++ dependencyClasspath def dependencies: Seq[Dependency] = dependenciesArray.to - def needsUpdate: Boolean = needsUpdateCompat } implicit class ContextExtensions(subject: Context){ import subject._ val paths = CbtPaths(cbtHome, cache) implicit def logger: Logger = new Logger(enabledLoggers, start) - def classLoaderCache: ClassLoaderCache = new ClassLoaderCache( logger, persistentCache ) - def cbtDependency = { + def classLoaderCache: ClassLoaderCache = new ClassLoaderCache( persistentCache ) + def cbtDependencies = { import paths._ - new CbtDependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, stage2Target, compatibilityTarget)(logger, transientCache) + new CbtDependencies(mavenCache, nailgunTarget, stage1Target, stage2Target, compatibilityTarget)(logger, transientCache) } + val cbtDependency = cbtDependencies.stage2Dependency + def args: Seq[String] = argsArray.to def enabledLoggers: Set[String] = enabledLoggersArray.to def scalaVersion = Option(scalaVersionOrNull) def parentBuild = Option(parentBuildOrNull) - def start: scala.Long = startCompat - def cbtHasChanged: scala.Boolean = cbtHasChangedCompat + def cbtLastModified: scala.Long = subject.cbtLastModified def copy( projectDirectory: File = projectDirectory, args: Seq[String] = args, //enabledLoggers: Set[String] = enabledLoggers, - cbtHasChanged: Boolean = cbtHasChanged, + cbtLastModified: Long = cbtLastModified, scalaVersion: Option[String] = scalaVersion, cbtHome: File = cbtHome, parentBuild: Option[BuildInterface] = None @@ -84,8 +89,8 @@ object `package`{ cwd, args.to, enabledLoggers.to, - startCompat, - cbtHasChangedCompat, + start, + cbtLastModified, scalaVersion.getOrElse(null), persistentCache, transientCache, diff --git a/stage1/resolver.scala b/stage1/resolver.scala index 8e46135..4a39d14 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -18,22 +18,14 @@ trait DependencyImplementation extends Dependency{ */ protected lazy val taskCache = new PerClassCache(transientCache, moduleKey) - /** - CAREFUL: this is never allowed to return true for the same dependency more than - once in a single cbt run. Otherwise we can end up with multiple classloaders - for the same classes since classLoaderRecursion recreates the classLoader when it - sees this flag and it can be called multiple times. Maybe we can find a safer - solution than this current state. - */ - def needsUpdate: Boolean - //def cacheClassLoader: Boolean = false private[cbt] def targetClasspath: ClassPath def dependencyClasspathArray: Array[File] = dependencyClasspath.files.toArray def exportedClasspathArray: Array[File] = exportedClasspath.files.toArray def exportedClasspath: ClassPath def dependenciesArray: Array[Dependency] = dependencies.to - def needsUpdateCompat: java.lang.Boolean = needsUpdate + @deprecated("this method is replaced by lastModifiedCompat","") + def needsUpdateCompat = true /* //private type BuildCache = KeyLockedLazyCache[Dependency, Future[ClassPath]] @@ -122,63 +114,63 @@ trait DependencyImplementation extends Dependency{ } // TODO: all this hard codes the scala version, needs more flexibility -class ScalaCompilerDependency(cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-compiler",version, Classifier.none), Seq(mavenCentral)) -class ScalaLibraryDependency (cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-library",version, Classifier.none), Seq(mavenCentral)) -class ScalaReflectDependency (cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-reflect",version, Classifier.none), Seq(mavenCentral)) +class ScalaCompilerDependency(cbtLastModified: Long, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BoundMavenDependency(cbtLastModified, mavenCache, MavenDependency("org.scala-lang","scala-compiler",version, Classifier.none), Seq(mavenCentral)) +class ScalaLibraryDependency (cbtLastModified: Long, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BoundMavenDependency(cbtLastModified, mavenCache, MavenDependency("org.scala-lang","scala-library",version, Classifier.none), Seq(mavenCentral)) +class ScalaReflectDependency (cbtLastModified: Long, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BoundMavenDependency(cbtLastModified, mavenCache, MavenDependency("org.scala-lang","scala-reflect",version, Classifier.none), Seq(mavenCentral)) -class ScalaDependencies(cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends Dependencies( +class ScalaDependencies(cbtLastModified: Long, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends Dependencies( Seq( - new ScalaCompilerDependency(cbtHasChanged, mavenCache, version), - new ScalaLibraryDependency(cbtHasChanged, mavenCache, version), - new ScalaReflectDependency(cbtHasChanged, mavenCache, version) + new ScalaCompilerDependency(cbtLastModified, mavenCache, version), + new ScalaLibraryDependency(cbtLastModified, mavenCache, version), + new ScalaReflectDependency(cbtLastModified, mavenCache, version) ) ) case class BinaryDependency( paths: Seq[File], dependencies: Seq[Dependency] )(implicit val logger: Logger, val transientCache: java.util.Map[AnyRef,AnyRef]) extends DependencyImplementation{ assert(paths.nonEmpty) def exportedClasspath = ClassPath(paths) - override def needsUpdate = false + override def lastModified = paths.map(_.lastModified).maxOption.getOrElse(0) // FIXME: cache this def targetClasspath = exportedClasspath def moduleKey = this.getClass.getName ++ "(" ++ paths.mkString(", ") ++ ")" } /** Allows to easily assemble a bunch of dependencies */ case class Dependencies( dependencies: Seq[Dependency] )(implicit val logger: Logger, val transientCache: java.util.Map[AnyRef,AnyRef]) extends DependencyImplementation{ - override def needsUpdate = dependencies.exists(_.needsUpdate) - override def exportedClasspath = ClassPath() - override def targetClasspath = ClassPath() + override def lastModified = dependencies.map(_.lastModified).maxOption.getOrElse(0) def moduleKey = this.getClass.getName ++ "(" ++ dependencies.map(_.moduleKey).mkString(", ") ++ ")" + def targetClasspath = ClassPath() + def exportedClasspath = ClassPath() } -class Stage1Dependency(cbtHasChanged: Boolean, mavenCache: File, nailgunTarget: File, stage1Target: File, compatibilityTarget: File)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BinaryDependency( - Seq(nailgunTarget, stage1Target), - Seq( - new CompatibilityDependency(cbtHasChanged, compatibilityTarget) - ) ++ - MavenResolver(cbtHasChanged,mavenCache,mavenCentral).bind( - MavenDependency("org.scala-lang","scala-library",constants.scalaVersion), - MavenDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,constants.scalaXmlVersion) - ) -){ - val compatibilityDependency = new CompatibilityDependency(cbtHasChanged, compatibilityTarget) - +case class PostBuildDependency(target: File, _dependencies: Seq[DependencyImplementation])(implicit val logger: Logger, val transientCache: java.util.Map[AnyRef,AnyRef]) extends DependencyImplementation{ + override final lazy val lastModified = (target++".last-success").lastModified + def moduleKey = target.string + override def targetClasspath = exportedClasspath + override def exportedClasspath = ClassPath( Seq(target) ) + override def dependencies = _dependencies } - -class CompatibilityDependency(cbtHasChanged: Boolean, compatibilityTarget: File)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BinaryDependency( - Seq(compatibilityTarget), Nil -) - -class CbtDependency(cbtHasChanged: Boolean, mavenCache: File, nailgunTarget: File, stage1Target: File, stage2Target: File, compatibilityTarget: File)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BinaryDependency( - Seq( stage2Target ), - Seq( - new Stage1Dependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, compatibilityTarget) - ) ++ - MavenResolver(cbtHasChanged, mavenCache,mavenCentral).bind( - MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0"), - MavenDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r") +case class CbtDependencies(mavenCache: File, nailgunTarget: File, stage1Target: File, stage2Target: File, compatibilityTarget: File)(implicit logger: Logger, val transientCache: java.util.Map[AnyRef,AnyRef]){ + val compatibilityDependency = PostBuildDependency(compatibilityTarget, Nil) + val cbtLastModified = (stage2Target++".last-success").lastModified + val stage1Dependency = PostBuildDependency( + stage1Target, + Seq( + PostBuildDependency(nailgunTarget, Nil), + compatibilityDependency + ) ++ + MavenResolver(cbtLastModified,mavenCache,mavenCentral).bind( + MavenDependency("org.scala-lang","scala-library",constants.scalaVersion), + MavenDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,constants.scalaXmlVersion) + ) + ) + val stage2Dependency = PostBuildDependency( + stage2Target, + stage1Dependency +: + MavenResolver(cbtLastModified, mavenCache,mavenCentral).bind( + MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0"), + MavenDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r") + ) ) -){ - val stage1Dependency = new Stage1Dependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, compatibilityTarget) } case class Classifier(name: Option[String]) @@ -191,11 +183,11 @@ abstract class DependenciesProxy{ } class BoundMavenDependencies( - cbtHasChanged: Boolean, mavenCache: File, urls: Seq[URL], mavenDependencies: Seq[MavenDependency] + cbtLastModified: Long, mavenCache: File, urls: Seq[URL], mavenDependencies: Seq[MavenDependency] )( implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef] ) extends Dependencies( - mavenDependencies.map( BoundMavenDependency(cbtHasChanged,mavenCache,_,urls) ) + mavenDependencies.map( BoundMavenDependency(cbtLastModified,mavenCache,_,urls) ) ) case class MavenDependency( groupId: String, artifactId: String, version: String, classifier: Classifier = Classifier.none @@ -209,7 +201,7 @@ object MavenDependency{ } // FIXME: take MavenResolver instead of mavenCache and repositories separately case class BoundMavenDependency( - cbtHasChanged: Boolean, mavenCache: File, mavenDependency: MavenDependency, repositories: Seq[URL] + cbtLastModified: Long, mavenCache: File, mavenDependency: MavenDependency, repositories: Seq[URL] )( implicit val logger: Logger, val transientCache: java.util.Map[AnyRef,AnyRef] ) extends ArtifactInfo with DependencyImplementation{ @@ -233,7 +225,7 @@ case class BoundMavenDependency( ) override def show: String = this.getClass.getSimpleName ++ "(" ++ mavenDependency.serialize ++ ")" - override def needsUpdate = false + override final lazy val lastModified = classpath.strings.map(new File(_).lastModified).max private val groupPath = groupId.split("\\.").mkString("/") protected[cbt] def basePath(useClassifier: Boolean) = s"/$groupPath/$artifactId/$version/$artifactId-$version" ++ (if (useClassifier) classifier.name.map("-"++_).getOrElse("") else "") @@ -276,7 +268,7 @@ case class BoundMavenDependency( (pomXml \ "parent").collect{ case parent => BoundMavenDependency( - cbtHasChanged: Boolean, + cbtLastModified: Long, mavenCache, MavenDependency( (parent \ "groupId").text, @@ -314,7 +306,7 @@ case class BoundMavenDependency( if(classifier == Classifier.sources) Seq() else { lib.cacheOnDisk( - cbtHasChanged, mavenCache ++ basePath(true) ++ ".pom.dependencies" + cbtLastModified, mavenCache ++ basePath(true) ++ ".pom.dependencies" )( MavenDependency.deserialize )( _.serialize ){ (pomXml \ "dependencies" \ "dependency").collect{ case xml if ( (xml \ "scope").text == "" || (xml \ "scope").text == "compile" ) && (xml \ "optional").text != "true" => @@ -339,7 +331,7 @@ case class BoundMavenDependency( MavenDependency( groupId, artifactId, version, classifier ) }.toVector }.map( - BoundMavenDependency( cbtHasChanged, mavenCache, _, repositories ) + BoundMavenDependency( cbtLastModified, mavenCache, _, repositories ) ).to } } diff --git a/stage2/BasicBuild.scala b/stage2/BasicBuild.scala index 2fd34c7..ef5411a 100644 --- a/stage2/BasicBuild.scala +++ b/stage2/BasicBuild.scala @@ -98,7 +98,7 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with Trigge } logEmptySourceDirectories() - def Resolver( urls: URL* ) = MavenResolver( context.cbtHasChanged, context.paths.mavenCache, urls: _* ) + def Resolver( urls: URL* ) = MavenResolver( context.cbtLastModified, context.paths.mavenCache, urls: _* ) def ScalaDependency( groupId: String, artifactId: String, version: String, classifier: Classifier = Classifier.none, @@ -126,7 +126,7 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with Trigge val resourcesDirectory = projectDirectory ++ "/resources" ClassPath( if(resourcesDirectory.exists) Seq(resourcesDirectory) else Nil ) } - def exportedClasspath : ClassPath = ClassPath(compile.toSeq) ++ resourceClasspath + def exportedClasspath : ClassPath = ClassPath(compileFile.toSeq) ++ resourceClasspath def targetClasspath = ClassPath(Seq(compileTarget)) // ========== compile, run, test ========== @@ -137,23 +137,20 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with Trigge "-unchecked" ) - def needsUpdate: Boolean = taskCache[BaseBuild]("needsUpdate").memoize[java.lang.Boolean]( - context.cbtHasChanged - || lib.needsUpdate( sourceFiles, compileStatusFile ) - || transitiveDependencies.filterNot(_ == context.parentBuild).exists(_.needsUpdate) - ) + final def lastModified: Long = compile.getOrElse(0L) + + final def compileFile: Option[File] = compile.map(_ => compileTarget) - def compile: Option[File] = taskCache[BaseBuild]("compile").memoize{ + def compile: Option[Long] = taskCache[BaseBuild]("_compile").memoize{ lib.compile( - context.cbtHasChanged, - needsUpdate || context.parentBuild.map(_.needsUpdate).getOrElse(false), + context.cbtLastModified, sourceFiles, compileTarget, compileStatusFile, compileDependencies, context.paths.mavenCache, scalacOptions, context.classLoaderCache, zincVersion = zincVersion, scalaVersion = scalaVersion ) } - def mainClasses: Seq[Class[_]] = compile.toSeq.flatMap( lib.mainClasses( _, classLoader(classLoaderCache) ) ) + def mainClasses: Seq[Class[_]] = compileFile.toSeq.flatMap( lib.mainClasses( _, classLoader(classLoaderCache) ) ) def runClass: Option[String] = lib.runClass( mainClasses ).map( _.getName ) @@ -181,7 +178,7 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with Trigge System.setProperty(colorized, "true") } - val scalac = new ScalaCompilerDependency(context.cbtHasChanged, context.paths.mavenCache, scalaVersion) + val scalac = new ScalaCompilerDependency(context.cbtLastModified, context.paths.mavenCache, scalaVersion) lib.runMain( "scala.tools.nsc.MainGenericRunner", Seq( diff --git a/stage2/BuildDependency.scala b/stage2/BuildDependency.scala index 4b4fdc1..236f958 100644 --- a/stage2/BuildDependency.scala +++ b/stage2/BuildDependency.scala @@ -27,7 +27,7 @@ final case class DirectoryDependency(context: Context) extends TriggerLoop{ def exportedClasspath = ClassPath() def dependencies = Seq(build) def triggerLoopFiles = root.triggerLoopFiles - def needsUpdate = build.needsUpdate + def lastModified = build.lastModified def targetClasspath = ClassPath() } /* diff --git a/stage2/GitDependency.scala b/stage2/GitDependency.scala index e27eff9..059d650 100644 --- a/stage2/GitDependency.scala +++ b/stage2/GitDependency.scala @@ -76,5 +76,5 @@ case class GitDependency( def exportedClasspath = ClassPath() private[cbt] def targetClasspath = exportedClasspath - def needsUpdate: Boolean = false + def lastModified: Long = dependency.lastModified } diff --git a/stage2/Lib.scala b/stage2/Lib.scala index c570ca3..a76e281 100644 --- a/stage2/Lib.scala +++ b/stage2/Lib.scala @@ -15,7 +15,7 @@ import scala.util._ case class Developer(id: String, name: String, timezone: String, url: URL) /** Don't extend. Create your own libs :). */ -final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ +final class Lib(val logger: Logger) extends Stage1Lib(logger) with Scaffold{ lib => val buildClassName = "Build" @@ -74,7 +74,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ } def docJar( - cbtHasChanged: Boolean, + cbtLastModified: Long, scalaVersion: String, sourceFiles: Seq[File], dependencyClasspath: ClassPath, @@ -101,7 +101,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ runMain( "scala.tools.nsc.ScalaDoc", args, - new ScalaDependencies(cbtHasChanged,mavenCache,scalaVersion).classLoader(classLoaderCache) + new ScalaDependencies(cbtLastModified,mavenCache,scalaVersion).classLoader(classLoaderCache) ) } lib.jarFile( diff --git a/stage2/PackageJars.scala b/stage2/PackageJars.scala index 3ecceb2..7079786 100644 --- a/stage2/PackageJars.scala +++ b/stage2/PackageJars.scala @@ -11,7 +11,7 @@ trait PackageJars extends BaseBuild with ArtifactInfo{ )( _() ).flatten def jar: Option[File] = taskCache[PackageJars]("jar").memoize{ - compile.flatMap( lib.jar( artifactId, scalaMajorVersion, version, _, jarTarget ) ) + compileFile.flatMap( lib.jar( artifactId, scalaMajorVersion, version, _, jarTarget ) ) } def srcJar: Option[File] = taskCache[PackageJars]("srcJar").memoize{ @@ -20,7 +20,7 @@ trait PackageJars extends BaseBuild with ArtifactInfo{ def docJar: Option[File] = taskCache[PackageJars]("docJar").memoize{ lib.docJar( - context.cbtHasChanged, + context.cbtLastModified, scalaVersion, sourceFiles, compileClasspath, docTarget, jarTarget, artifactId, scalaMajorVersion, version, scalacOptions, context.classLoaderCache, context.paths.mavenCache diff --git a/stage2/Stage2.scala b/stage2/Stage2.scala index 2884ddb..542a982 100644 --- a/stage2/Stage2.scala +++ b/stage2/Stage2.scala @@ -30,9 +30,9 @@ object Stage2 extends Stage2Base{ args.args.drop( taskIndex +1 ).toArray, logger.enabledLoggers.toArray, logger.start, - args.cbtHasChanged, + args.stage2LastModified, null, - args.persistentCache, + args.classLoaderCache.hashMap, args.transientCache, args.cache, args.cbtHome, diff --git a/stage2/ToolsStage2.scala b/stage2/ToolsStage2.scala index df615fc..2b5e092 100644 --- a/stage2/ToolsStage2.scala +++ b/stage2/ToolsStage2.scala @@ -4,7 +4,7 @@ object ToolsStage2 extends Stage2Base{ def run( _args: Stage2Args ): Unit = { val args = _args.args.dropWhile(Seq("tools","direct") contains _) val lib = new Lib(_args.logger) - val toolsTasks = new ToolsTasks(lib, args, _args.cwd, _args.classLoaderCache, _args.cache, _args.cbtHome, _args.cbtHasChanged) + val toolsTasks = new ToolsTasks(lib, args, _args.cwd, _args.classLoaderCache, _args.cache, _args.cbtHome, _args.stage2LastModified) new lib.ReflectObject(toolsTasks){ def usage: String = "Available methods: " ++ lib.taskNames(toolsTasks.getClass).mkString(" ") }.callNullary(args.lift(0)) diff --git a/stage2/ToolsTasks.scala b/stage2/ToolsTasks.scala index 839780a..6acf72c 100644 --- a/stage2/ToolsTasks.scala +++ b/stage2/ToolsTasks.scala @@ -9,11 +9,11 @@ class ToolsTasks( classLoaderCache: ClassLoaderCache, cache: File, cbtHome: File, - cbtHasChanged: Boolean + cbtLastModified: Long ){ private val paths = CbtPaths(cbtHome, cache) import paths._ - private def Resolver( urls: URL* ) = MavenResolver(cbtHasChanged,mavenCache,urls: _*) + private def Resolver( urls: URL* ) = MavenResolver(cbtLastModified,mavenCache,urls: _*) implicit val logger: Logger = lib.logger implicit val transientCache: java.util.Map[AnyRef,AnyRef] = new java.util.HashMap def createMain: Unit = lib.createMain( cwd ) @@ -56,7 +56,7 @@ class ToolsTasks( } def scala = { val version = args.lift(1).getOrElse(constants.scalaVersion) - val scalac = new ScalaCompilerDependency( cbtHasChanged, mavenCache, version ) + val scalac = new ScalaCompilerDependency( cbtLastModified, mavenCache, version ) val _args = Seq("-cp", scalac.classpath.string) ++ args.drop(2) lib.runMain( "scala.tools.nsc.MainGenericRunner", _args, scalac.classLoader(classLoaderCache) @@ -97,14 +97,15 @@ class ToolsTasks( val n = valName(d) s""" // ${d.groupId}:${d.artifactId}:${d.version} - download(new URL(mavenUrl + "${d.basePath(true)}.jar"), Paths.get(${n}File), "${d.jarSha1}"); - String[] ${n}ClasspathArray = new String[]{${deps.sortBy(_.jar).map(valName(_)+"File").mkString(", ")}}; - String ${n}Classpath = classpath( ${n}ClasspathArray ); - ClassLoader $n = - classLoaderCache.contains( ${n}Classpath ) - ? classLoaderCache.get( ${n}Classpath ) - : classLoaderCache.put( classLoader( ${n}File, $parentString ), ${n}Classpath );""" + ClassLoader $n = loadDependency( + mavenUrl + "${d.basePath(true)}.jar", + ${n}File, + "${d.jarSha1}", + classLoaderCache, + $parentString, + ${n}ClasspathArray + );""" } } val assignments = codeEach(zinc) ++ codeEach(scalaXml) @@ -116,6 +117,7 @@ import java.io.*; import java.nio.file.*; import java.net.*; import java.security.*; +import java.util.*; import static cbt.Stage0Lib.*; import static cbt.NailgunLauncher.*; @@ -130,13 +132,13 @@ class EarlyDependencies{ ${files.map(d => s""" String ${valName(d)}File;""").mkString("\n")} public EarlyDependencies( - String mavenCache, String mavenUrl, JavaCache classLoaderCache, ClassLoader rootClassLoader + String mavenCache, String mavenUrl, ClassLoaderCache classLoaderCache, ClassLoader rootClassLoader ) throws Throwable { ${files.map(d => s""" ${valName(d)}File = mavenCache + "${d.basePath(true)}.jar";""").mkString("\n")} ${scalaDeps.map(d => s""" download(new URL(mavenUrl + "${d.basePath(true)}.jar"), Paths.get(${valName(d)}File), "${d.jarSha1}");""").mkString("\n")} ${assignments.mkString("\n")} - + classLoader = scalaXml_${scalaXmlVersion.replace(".","_")}_; classpathArray = scalaXml_${scalaXmlVersion.replace(".","_")}_ClasspathArray; diff --git a/stage2/plugins/Dotty.scala b/stage2/plugins/Dotty.scala index fe949a3..6fe5dd3 100644 --- a/stage2/plugins/Dotty.scala +++ b/stage2/plugins/Dotty.scala @@ -8,17 +8,15 @@ trait Dotty extends BaseBuild{ def dottyVersion: String = "0.1-20160926-ec28ea1-NIGHTLY" def dottyOptions: Seq[String] = Seq() override def scalaTarget: File = target ++ s"/dotty-$dottyVersion" - + private lazy val dottyLib = new DottyLib( - logger, context.cbtHasChanged, context.paths.mavenCache, + logger, context.cbtLastModified, context.paths.mavenCache, context.classLoaderCache, dottyVersion = dottyVersion ) - override def compile: Option[File] = taskCache[Dotty]("compile").memoize{ + override def compile: Option[Long] = taskCache[Dotty]("compile").memoize{ dottyLib.compile( - needsUpdate || context.parentBuild.map(_.needsUpdate).getOrElse(false), - sourceFiles, compileTarget, compileStatusFile, compileClasspath, - dottyOptions + sourceFiles, compileTarget, compileStatusFile, compileDependencies, dottyOptions ) } @@ -36,7 +34,7 @@ trait Dotty extends BaseBuild{ class DottyLib( logger: Logger, - cbtHasChanged: Boolean, + cbtLastModified: Long, mavenCache: File, classLoaderCache: ClassLoaderCache, dottyVersion: String @@ -44,7 +42,7 @@ class DottyLib( val lib = new Lib(logger) import lib._ - private def Resolver(urls: URL*) = MavenResolver(cbtHasChanged, mavenCache, urls: _*) + private def Resolver(urls: URL*) = MavenResolver(cbtLastModified, mavenCache, urls: _*) private lazy val dottyDependency = Resolver(mavenCentral).bindOne( MavenDependency("ch.epfl.lamp","dotty_2.11",dottyVersion) ) @@ -94,22 +92,24 @@ class DottyLib( } def compile( - needsRecompile: Boolean, - files: Seq[File], + sourceFiles: Seq[File], compileTarget: File, statusFile: File, - classpath: ClassPath, + dependencies: Seq[Dependency], dottyOptions: Seq[String] - ): Option[File] = { - + ): Option[Long] = { + val d = Dependencies(dependencies) + val classpath = d.classpath + val cp = classpath.string if(classpath.files.isEmpty) - throw new Exception("Trying to compile with empty classpath. Source files: " ++ files.toString) + throw new Exception("Trying to compile with empty classpath. Source files: " ++ sourceFiles.toString) - if( files.isEmpty ){ + if( sourceFiles.isEmpty ){ None }else{ - if( needsRecompile ){ - val start = System.currentTimeMillis + val start = System.currentTimeMillis + val lastCompiled = statusFile.lastModified + if( d.lastModified > lastCompiled || sourceFiles.exists(_.lastModified > lastCompiled) ){ val _class = "dotty.tools.dotc.Main" val dualArgs = @@ -118,7 +118,7 @@ class DottyLib( ) val singleArgs = dottyOptions.map( "-S" ++ _ ) - val code = + val code = try{ System.err.println("Compiling with Dotty to " ++ compileTarget.toString) compileTarget.mkdirs @@ -128,7 +128,7 @@ class DottyLib( dualArgs ++ singleArgs ++ Seq( "-bootclasspath", dottyDependency.classpath.string, // let's put cp last. It so long "-classpath", classpath.string // let's put cp last. It so long - ) ++ files.map(_.toString), + ) ++ sourceFiles.map(_.toString), dottyDependency.classLoader(classLoaderCache) ) } @@ -150,7 +150,7 @@ ${dottyDependency.classpath.strings.mkString(":\\\n")} \\ -classpath \\ ${classpath.strings.mkString(":\\\n")} \\ \\ -${files.sorted.mkString(" \\\n")} +${sourceFiles.sorted.mkString(" \\\n")} """ ) ExitCode.Failure @@ -164,8 +164,10 @@ ${files.sorted.mkString(" \\\n")} } else { System.exit(code.integer) // FIXME: let's find a better solution for error handling. Maybe a monad after all. } + Some( start ) + } else { + Some( lastCompiled ) } - Some( compileTarget ) } } } diff --git a/test/test.scala b/test/test.scala index 332b61e..e9da8bf 100644 --- a/test/test.scala +++ b/test/test.scala @@ -103,7 +103,7 @@ object Main{ val cache = cbtHome ++ "/cache" val mavenCache = cache ++ "/maven" - val cbtHasChanged = true + val cbtLastModified = System.currentTimeMillis implicit val transientCache: java.util.Map[AnyRef,AnyRef] = new java.util.HashMap def Resolver(urls: URL*) = MavenResolver(cbtLastModified, mavenCache, urls: _*) @@ -114,7 +114,7 @@ object Main{ Array(), Array(), start, - cbtHasChanged, + cbtLastModified, null, new HashMap[AnyRef,AnyRef], new HashMap[AnyRef,AnyRef], -- cgit v1.2.3