From 00d9485f5597fdecc58461bd81df635fafbe494f Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Fri, 25 Nov 2016 16:48:28 -0500 Subject: Merge separate hashmaps for persistent cache into one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This isn’t type-safe, but re-using that same hashmap for both keys and classloaders allows to reduce the number of members in Context. Also we can re-use the same hashMap for other things as well in the coming commits, e.g. timestamps. --- stage2/BasicBuild.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'stage2/BasicBuild.scala') diff --git a/stage2/BasicBuild.scala b/stage2/BasicBuild.scala index 7ff1f4b..889e32d 100644 --- a/stage2/BasicBuild.scala +++ b/stage2/BasicBuild.scala @@ -272,7 +272,7 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with Trigge the context is fresh on every complete run of cbt */ def cached[T <: AnyRef](name: String)(task: => T): T = { - val cache = context.taskCache + val cache = context.transientCache val key = (projectDirectory,name) if( cache.containsKey(key) ){ cache.get(key).asInstanceOf[T] -- cgit v1.2.3 From 0d98155071d75dd135adca9d5977b365088f8190 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Wed, 1 Feb 2017 23:53:08 -0500 Subject: make full dependencies available to compile instead of only classpath this will make it possible to access lastModified times and cache them in the following commits --- stage1/Stage1.scala | 2 +- stage1/Stage1Lib.scala | 3 ++- stage1/resolver.scala | 4 ++-- stage2/BasicBuild.scala | 11 +++++------ 4 files changed, 10 insertions(+), 10 deletions(-) (limited to 'stage2/BasicBuild.scala') diff --git a/stage1/Stage1.scala b/stage1/Stage1.scala index 2f8f960..f0540b4 100644 --- a/stage1/Stage1.scala +++ b/stage1/Stage1.scala @@ -98,7 +98,7 @@ object Stage1{ cbtHasChanged, cbtHasChanged, stage2sourceFiles, stage2Target, stage2StatusFile, - cbtDependency.dependencyClasspath, + cbtDependency.dependencies, mavenCache, Seq("-deprecation","-feature","-unchecked"), classLoaderCache, zincVersion = constants.zincVersion, scalaVersion = constants.scalaVersion diff --git a/stage1/Stage1Lib.scala b/stage1/Stage1Lib.scala index 9b48409..296581c 100644 --- a/stage1/Stage1Lib.scala +++ b/stage1/Stage1Lib.scala @@ -194,7 +194,7 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ files: Seq[File], compileTarget: File, statusFile: File, - classpath: ClassPath, + dependencies: Seq[Dependency], mavenCache: File, scalacOptions: Seq[String] = Seq(), classLoaderCache: ClassLoaderCache, @@ -202,6 +202,7 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ scalaVersion: String ): Option[File] = { + val classpath = Dependencies(dependencies).classpath val cp = classpath.string if(classpath.files.isEmpty) throw new Exception("Trying to compile with empty classpath. Source files: " ++ files.toString) diff --git a/stage1/resolver.scala b/stage1/resolver.scala index 1f94c7f..f4af73e 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -128,8 +128,8 @@ case class ScalaDependencies(cbtHasChanged: Boolean, mavenCache: File, version: ) } -case class BinaryDependency( path: File, dependencies: Seq[Dependency] )(implicit val logger: Logger) extends DependencyImplementation{ - def exportedClasspath = ClassPath(Seq(path)) +case class BinaryDependency( paths: Seq[File], dependencies: Seq[Dependency] )(implicit val logger: Logger) extends DependencyImplementation{ + def exportedClasspath = ClassPath(paths) override def needsUpdate = false def targetClasspath = exportedClasspath } diff --git a/stage2/BasicBuild.scala b/stage2/BasicBuild.scala index 889e32d..8b4a3a5 100644 --- a/stage2/BasicBuild.scala +++ b/stage2/BasicBuild.scala @@ -59,7 +59,7 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with Trigge // FIXME: this should probably be removed Resolver( mavenCentral ).bind( "org.scala-lang" % "scala-library" % scalaVersion - ) + ) :+ BinaryDependency(localJars, Nil) // ========== paths ========== final private val defaultSourceDirectory = projectDirectory ++ "/src" @@ -115,11 +115,10 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with Trigge .flatMap(_.listFiles) .filter(_.toString.endsWith(".jar")) - override def dependencyClasspath : ClassPath = ClassPath(localJars) ++ super.dependencyClasspath + override def dependencyClasspath : ClassPath = super.dependencyClasspath - protected def compileDependencies: Seq[Dependency] = Nil - final def compileClasspath : ClassPath = - dependencyClasspath ++ ClassPath( compileDependencies.flatMap(_.exportedClasspath.files).distinct ) + protected def compileDependencies: Seq[Dependency] = dependencies + final def compileClasspath : ClassPath = Dependencies(compileDependencies).classpath def resourceClasspath: ClassPath = { val resourcesDirectory = projectDirectory ++ "/resources" @@ -148,7 +147,7 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with Trigge lib.compile( context.cbtHasChanged, needsUpdate || context.parentBuild.map(_.needsUpdate).getOrElse(false), - sourceFiles, compileTarget, compileStatusFile, compileClasspath, + sourceFiles, compileTarget, compileStatusFile, compileDependencies, context.paths.mavenCache, scalacOptions, context.classLoaderCache, zincVersion = zincVersion, scalaVersion = scalaVersion ) -- cgit v1.2.3 From bee13ba7a4458482ce00a5c6bae4cd64328c4e5e Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Thu, 9 Feb 2017 02:29:44 +0000 Subject: memoize task results across classes within a single run --- compatibility/Dependency.java | 1 + plugins/essentials/DynamicOverrides.scala | 10 +-- plugins/scalajs/ScalaJsLib.scala | 2 +- stage1/Cache.scala | 14 ---- stage1/ContextImplementation.scala | 32 +++++----- stage1/MavenRepository.scala | 10 ++- stage1/Stage1.scala | 15 +++-- stage1/Stage1Lib.scala | 25 +++++++- stage1/cbt.scala | 4 +- stage1/logger.scala | 2 + stage1/resolver.scala | 102 ++++++++++++++++-------------- stage2/BasicBuild.scala | 29 ++------- stage2/BuildBuild.scala | 3 +- stage2/BuildDependency.scala | 3 + stage2/GitDependency.scala | 10 ++- stage2/Lib.scala | 4 +- stage2/PackageJars.scala | 9 +-- stage2/Stage2.scala | 4 +- stage2/ToolsTasks.scala | 1 + stage2/plugins/Dotty.scala | 5 +- test/test.scala | 5 +- 21 files changed, 148 insertions(+), 142 deletions(-) delete mode 100644 stage1/Cache.scala (limited to 'stage2/BasicBuild.scala') diff --git a/compatibility/Dependency.java b/compatibility/Dependency.java index d491174..efb9214 100644 --- a/compatibility/Dependency.java +++ b/compatibility/Dependency.java @@ -3,6 +3,7 @@ import java.io.*; public interface Dependency{ public abstract String show(); + public abstract String moduleKey(); public abstract Boolean needsUpdateCompat(); public abstract Dependency[] dependenciesArray(); public abstract File[] dependencyClasspathArray(); diff --git a/plugins/essentials/DynamicOverrides.scala b/plugins/essentials/DynamicOverrides.scala index 4a3fe9e..1050f98 100644 --- a/plugins/essentials/DynamicOverrides.scala +++ b/plugins/essentials/DynamicOverrides.scala @@ -1,10 +1,12 @@ package cbt import cbt.eval.Eval trait DynamicOverrides extends BaseBuild{ - private val twitterEval = cached("eval"){ - new Eval{ - override lazy val impliedClassPath: List[String] = context.parentBuild.get.classpath.strings.toList//new ScalaCompilerDependency( context.cbtHasChanged, context.paths.mavenCache, scalaVersion ).classpath.strings.toList - override def classLoader = DynamicOverrides.this.getClass.getClassLoader + private val twitterEval = { + taskCache[DynamicOverrides]( "eval" ).memoize{ + new Eval{ + override lazy val impliedClassPath: List[String] = context.parentBuild.get.classpath.strings.toList//new ScalaCompilerDependency( context.cbtLastModified, context.paths.mavenCache, scalaVersion ).classpath.strings.toList + override def classLoader = DynamicOverrides.this.getClass.getClassLoader + } } } diff --git a/plugins/scalajs/ScalaJsLib.scala b/plugins/scalajs/ScalaJsLib.scala index 0355850..12c1c85 100644 --- a/plugins/scalajs/ScalaJsLib.scala +++ b/plugins/scalajs/ScalaJsLib.scala @@ -4,7 +4,7 @@ import java.io.File case class ScalaJsLib( scalaJsVersion: String, scalaVersion: String, cbtHasChanged: Boolean, classLoaderCache: ClassLoaderCache, mavenCache: File -)(implicit logger: Logger){ +)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]){ sealed trait ScalaJsOutputMode { def option: String def fileSuffix: String diff --git a/stage1/Cache.scala b/stage1/Cache.scala deleted file mode 100644 index a8036e5..0000000 --- a/stage1/Cache.scala +++ /dev/null @@ -1,14 +0,0 @@ -package cbt -/** -Caches exactly one value. -Is there a less boiler-platy way to achieve this, that doesn't -require creating an instance for each thing you want to cache? -*/ -class Cache[T]{ - private var value: Option[T] = None - def apply(value: => T) = this.synchronized{ - if(!this.value.isDefined) - this.value = Some(value) - this.value.get - } -} diff --git a/stage1/ContextImplementation.scala b/stage1/ContextImplementation.scala index 3b610c0..30db597 100644 --- a/stage1/ContextImplementation.scala +++ b/stage1/ContextImplementation.scala @@ -2,19 +2,19 @@ package cbt import java.io._ import java.lang._ -case class ContextImplementation( - projectDirectory: File, - cwd: File, - argsArray: Array[String], - enabledLoggersArray: Array[String], - startCompat: Long, - cbtHasChangedCompat: Boolean, - scalaVersionOrNull: String, - persistentCache: java.util.Map[AnyRef,AnyRef], - transientCache: java.util.Map[AnyRef,AnyRef], - cache: File, - cbtHome: File, - cbtRootHome: File, - compatibilityTarget: File, - parentBuildOrNull: BuildInterface -) extends Context \ No newline at end of file +class ContextImplementation( + val projectDirectory: File, + val cwd: File, + val argsArray: Array[String], + val enabledLoggersArray: Array[String], + val startCompat: Long, + val cbtHasChangedCompat: Boolean, + val scalaVersionOrNull: String, + val persistentCache: java.util.Map[AnyRef,AnyRef], + val transientCache: java.util.Map[AnyRef,AnyRef], + val cache: File, + val cbtHome: File, + val cbtRootHome: File, + val compatibilityTarget: File, + val parentBuildOrNull: BuildInterface +) extends Context diff --git a/stage1/MavenRepository.scala b/stage1/MavenRepository.scala index 4184d2d..2ac7064 100644 --- a/stage1/MavenRepository.scala +++ b/stage1/MavenRepository.scala @@ -1,9 +1,13 @@ package cbt import java.io._ import java.net._ -case class MavenResolver( cbtHasChanged: Boolean, mavenCache: File, urls: URL* ){ - def bind( dependencies: MavenDependency* )(implicit logger: Logger): Seq[BoundMavenDependency] +case class MavenResolver( + cbtHasChanged: Boolean, mavenCache: File, urls: URL* +)( + implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef] +){ + def bind( dependencies: MavenDependency* ): Seq[BoundMavenDependency] = dependencies.map( BoundMavenDependency(cbtHasChanged,mavenCache,_,urls.to) ).to - def bindOne( dependency: MavenDependency )(implicit logger: Logger): BoundMavenDependency + def bindOne( dependency: MavenDependency ): BoundMavenDependency = BoundMavenDependency( cbtHasChanged, mavenCache, dependency, urls.to ) } diff --git a/stage1/Stage1.scala b/stage1/Stage1.scala index f0540b4..27c6402 100644 --- a/stage1/Stage1.scala +++ b/stage1/Stage1.scala @@ -44,6 +44,8 @@ case class Stage2Args( cache: File, cbtHome: File, compatibilityTarget: File +)( + implicit val transientCache: java.util.Map[AnyRef,AnyRef] ){ val ClassLoaderCache( logger, persistentCache ) = classLoaderCache } @@ -60,7 +62,7 @@ object Stage1{ ClassLoaderCache( logger, context.persistentCache ), context.cbtHome, context.cache - ) + )( context.transientCache ) classLoader .loadClass("cbt.Stage2") @@ -75,7 +77,7 @@ object Stage1{ def buildStage2( buildStage1: BuildStage1Result, classLoaderCache: ClassLoaderCache, cbtHome: File, cache: File - ): (Boolean, ClassLoader) = { + )(implicit transientCache: java.util.Map[AnyRef,AnyRef]): (Boolean, ClassLoader) = { import classLoaderCache.logger val lib = new Stage1Lib(logger) @@ -86,12 +88,12 @@ object Stage1{ val stage2sourceFiles = ( stage2.listFiles ++ (stage2 ++ "/plugins").listFiles ).toVector.filter(_.isFile).filter(_.toString.endsWith(".scala")) - + val cbtHasChanged = buildStage1.changed || lib.needsUpdate(stage2sourceFiles, stage2StatusFile) val cls = this.getClass.getClassLoader.loadClass("cbt.NailgunLauncher") - - val cbtDependency = CbtDependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, stage2Target, new File(buildStage1.compatibilityClasspath)) + + val cbtDependency = new CbtDependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, stage2Target, new File(buildStage1.compatibilityClasspath)) logger.stage1("Compiling stage2 if necessary") compile( @@ -102,7 +104,7 @@ object Stage1{ mavenCache, Seq("-deprecation","-feature","-unchecked"), classLoaderCache, zincVersion = constants.zincVersion, scalaVersion = constants.scalaVersion - ) + )(transientCache) logger.stage1(s"calling CbtDependency.classLoader") if( cbtHasChanged && classLoaderCache.cache.containsKey( cbtDependency.classpath.string ) ) { @@ -163,6 +165,7 @@ object Stage1{ ): Int = { val args = Stage1ArgsParser(_args.toVector) val logger = new Logger(args.enabledLoggers, start) + implicit val transientCache: java.util.Map[AnyRef,AnyRef] = new java.util.HashMap logger.stage1(s"Stage1 start") val classLoaderCache = ClassLoaderCache( logger, persistentCache ) diff --git a/stage1/Stage1Lib.scala b/stage1/Stage1Lib.scala index 296581c..f0bb588 100644 --- a/stage1/Stage1Lib.scala +++ b/stage1/Stage1Lib.scala @@ -200,8 +200,9 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ classLoaderCache: ClassLoaderCache, zincVersion: String, scalaVersion: String + )( + implicit transientCache: java.util.Map[AnyRef, AnyRef] ): Option[File] = { - val classpath = Dependencies(dependencies).classpath val cp = classpath.string if(classpath.files.isEmpty) @@ -447,4 +448,26 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ else cache.cache.get( a.classpath.string, cl ).asInstanceOf[ClassLoader] } + +} + +import scala.reflect._ +import scala.language.existentials +case class PerClassCache(cache: java.util.Map[AnyRef,AnyRef], moduleKey: String)(implicit logger: Logger){ + def apply[D <: Dependency: ClassTag](key: AnyRef): MethodCache[D] = new MethodCache[D](key) + case class MethodCache[D <: Dependency: ClassTag](key: AnyRef){ + def memoize[T <: AnyRef](task: => T): T = { + val fullKey = (classTag[D].runtimeClass, moduleKey, key) + logger.transientCache("fetching key"+fullKey) + if( cache.containsKey(fullKey) ){ + logger.transientCache("found key"+fullKey) + cache.get(fullKey).asInstanceOf[T] + } else{ + val value = task + logger.transientCache("put key"+fullKey) + cache.put( fullKey, value ) + value + } + } + } } diff --git a/stage1/cbt.scala b/stage1/cbt.scala index 22242d7..01c9303 100644 --- a/stage1/cbt.scala +++ b/stage1/cbt.scala @@ -62,7 +62,7 @@ object `package`{ def classLoaderCache: ClassLoaderCache = new ClassLoaderCache( logger, persistentCache ) def cbtDependency = { import paths._ - CbtDependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, stage2Target, compatibilityTarget) + new CbtDependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, stage2Target, compatibilityTarget)(logger, transientCache) } def args: Seq[String] = argsArray.to def enabledLoggers: Set[String] = enabledLoggersArray.to @@ -79,7 +79,7 @@ object `package`{ scalaVersion: Option[String] = scalaVersion, cbtHome: File = cbtHome, parentBuild: Option[BuildInterface] = None - ): Context = ContextImplementation( + ): Context = new ContextImplementation( projectDirectory, cwd, args.to, diff --git a/stage1/logger.scala b/stage1/logger.scala index effdc35..8c8431a 100644 --- a/stage1/logger.scala +++ b/stage1/logger.scala @@ -41,6 +41,7 @@ case class Logger(enabledLoggers: Set[String], start: Long) { final def git(msg: => String) = log(names.git, msg) final def pom(msg: => String) = log(names.pom, msg) final def dynamic(msg: => String) = log(names.dynamic, msg) + final def transientCache(msg: => String) = log(names.transientCache, msg) private object names{ val stage1 = "stage1" @@ -54,6 +55,7 @@ case class Logger(enabledLoggers: Set[String], start: Long) { val pom = "pom" val git = "git" val dynamic = "dynamic" + val transientCache = "transientCache" } private def logUnguarded(name: String, msg: => String) = { diff --git a/stage1/resolver.scala b/stage1/resolver.scala index f4af73e..8e46135 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -8,6 +8,15 @@ import scala.xml._ trait DependencyImplementation extends Dependency{ implicit protected def logger: Logger protected def lib = new Stage1Lib(logger) + implicit protected def transientCache: java.util.Map[AnyRef,AnyRef] + + /** key used by taskCache to identify different objects that represent the same logical module */ + protected def moduleKey: String + /** + caches given value in context keyed with given key and projectDirectory + the context is fresh on every complete run of cbt + */ + protected lazy val taskCache = new PerClassCache(transientCache, moduleKey) /** CAREFUL: this is never allowed to return true for the same dependency more than @@ -101,11 +110,11 @@ trait DependencyImplementation extends Dependency{ ) def dependencies: Seq[Dependency] - private object transitiveDependenciesCache extends Cache[Seq[Dependency]] /** return dependencies in order of linearized dependence. this is a bit tricky. */ - def transitiveDependencies: Seq[Dependency] = transitiveDependenciesCache{ - lib.transitiveDependencies(this) - } + def transitiveDependencies: Seq[Dependency] = + taskCache[DependencyImplementation]( "transitiveDependencies" ).memoize{ + lib.transitiveDependencies(this) + } override def show: String = this.getClass.getSimpleName // ========== debug ========== @@ -113,65 +122,63 @@ trait DependencyImplementation extends Dependency{ } // TODO: all this hard codes the scala version, needs more flexibility -class ScalaCompilerDependency(cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-compiler",version, Classifier.none), Seq(mavenCentral)) -class ScalaLibraryDependency (cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-library",version, Classifier.none), Seq(mavenCentral)) -class ScalaReflectDependency (cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-reflect",version, Classifier.none), Seq(mavenCentral)) +class ScalaCompilerDependency(cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-compiler",version, Classifier.none), Seq(mavenCentral)) +class ScalaLibraryDependency (cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-library",version, Classifier.none), Seq(mavenCentral)) +class ScalaReflectDependency (cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-reflect",version, Classifier.none), Seq(mavenCentral)) -case class ScalaDependencies(cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit val logger: Logger) extends DependencyImplementation{ sd => - override final val needsUpdate = false - def targetClasspath = ClassPath() - def exportedClasspath = ClassPath() - def dependencies = Seq( +class ScalaDependencies(cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends Dependencies( + Seq( new ScalaCompilerDependency(cbtHasChanged, mavenCache, version), new ScalaLibraryDependency(cbtHasChanged, mavenCache, version), new ScalaReflectDependency(cbtHasChanged, mavenCache, version) ) -} +) -case class BinaryDependency( paths: Seq[File], dependencies: Seq[Dependency] )(implicit val logger: Logger) extends DependencyImplementation{ +case class BinaryDependency( paths: Seq[File], dependencies: Seq[Dependency] )(implicit val logger: Logger, val transientCache: java.util.Map[AnyRef,AnyRef]) extends DependencyImplementation{ + assert(paths.nonEmpty) def exportedClasspath = ClassPath(paths) override def needsUpdate = false def targetClasspath = exportedClasspath + def moduleKey = this.getClass.getName ++ "(" ++ paths.mkString(", ") ++ ")" } /** Allows to easily assemble a bunch of dependencies */ -case class Dependencies( dependencies: Seq[Dependency] )(implicit val logger: Logger) extends DependencyImplementation{ +case class Dependencies( dependencies: Seq[Dependency] )(implicit val logger: Logger, val transientCache: java.util.Map[AnyRef,AnyRef]) extends DependencyImplementation{ override def needsUpdate = dependencies.exists(_.needsUpdate) override def exportedClasspath = ClassPath() override def targetClasspath = ClassPath() + def moduleKey = this.getClass.getName ++ "(" ++ dependencies.map(_.moduleKey).mkString(", ") ++ ")" } -case class Stage1Dependency(cbtHasChanged: Boolean, mavenCache: File, nailgunTarget: File, stage1Target: File, compatibilityTarget: File)(implicit val logger: Logger) extends DependencyImplementation{ - override def needsUpdate = false - override def targetClasspath = exportedClasspath - override def exportedClasspath = ClassPath( Seq(nailgunTarget, stage1Target) ) - val compatibilityDependency = CompatibilityDependency(cbtHasChanged, compatibilityTarget) - override def dependencies = Seq( - compatibilityDependency +class Stage1Dependency(cbtHasChanged: Boolean, mavenCache: File, nailgunTarget: File, stage1Target: File, compatibilityTarget: File)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BinaryDependency( + Seq(nailgunTarget, stage1Target), + Seq( + new CompatibilityDependency(cbtHasChanged, compatibilityTarget) ) ++ MavenResolver(cbtHasChanged,mavenCache,mavenCentral).bind( MavenDependency("org.scala-lang","scala-library",constants.scalaVersion), MavenDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,constants.scalaXmlVersion) ) +){ + val compatibilityDependency = new CompatibilityDependency(cbtHasChanged, compatibilityTarget) + } -case class CompatibilityDependency(cbtHasChanged: Boolean, compatibilityTarget: File)(implicit val logger: Logger) extends DependencyImplementation{ - override def needsUpdate = false - override def targetClasspath = exportedClasspath - override def exportedClasspath = ClassPath( Seq(compatibilityTarget) ) - override def dependencies = Seq() -} -case class CbtDependency(cbtHasChanged: Boolean, mavenCache: File, nailgunTarget: File, stage1Target: File, stage2Target: File, compatibilityTarget: File)(implicit val logger: Logger) extends DependencyImplementation{ - override def needsUpdate = false - override def targetClasspath = exportedClasspath - override def exportedClasspath = ClassPath( Seq( stage2Target ) ) - val stage1Dependency = Stage1Dependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, compatibilityTarget) - override def dependencies = Seq( - stage1Dependency + +class CompatibilityDependency(cbtHasChanged: Boolean, compatibilityTarget: File)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BinaryDependency( + Seq(compatibilityTarget), Nil +) + +class CbtDependency(cbtHasChanged: Boolean, mavenCache: File, nailgunTarget: File, stage1Target: File, stage2Target: File, compatibilityTarget: File)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BinaryDependency( + Seq( stage2Target ), + Seq( + new Stage1Dependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, compatibilityTarget) ) ++ MavenResolver(cbtHasChanged, mavenCache,mavenCentral).bind( MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0"), MavenDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r") ) +){ + val stage1Dependency = new Stage1Dependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, compatibilityTarget) } case class Classifier(name: Option[String]) @@ -185,7 +192,9 @@ abstract class DependenciesProxy{ } class BoundMavenDependencies( cbtHasChanged: Boolean, mavenCache: File, urls: Seq[URL], mavenDependencies: Seq[MavenDependency] -)(implicit logger: Logger) extends Dependencies( +)( + implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef] +) extends Dependencies( mavenDependencies.map( BoundMavenDependency(cbtHasChanged,mavenCache,_,urls) ) ) case class MavenDependency( @@ -201,7 +210,10 @@ object MavenDependency{ // FIXME: take MavenResolver instead of mavenCache and repositories separately case class BoundMavenDependency( cbtHasChanged: Boolean, mavenCache: File, mavenDependency: MavenDependency, repositories: Seq[URL] -)(implicit val logger: Logger) extends ArtifactInfo with DependencyImplementation{ +)( + implicit val logger: Logger, val transientCache: java.util.Map[AnyRef,AnyRef] +) extends ArtifactInfo with DependencyImplementation{ + def moduleKey = this.getClass.getName ++ "(" ++ mavenDependency.serialize ++ ")" val MavenDependency( groupId, artifactId, version, classifier ) = mavenDependency assert( Option(groupId).collect{ @@ -251,17 +263,11 @@ case class BoundMavenDependency( StandardCharsets.UTF_8 ).mkString("\n").split(" ").head.trim } - - private object jarSha1Cache extends Cache[String] - def jarSha1: String = jarSha1Cache{ resolveHash("jar", true) } - - private object pomSha1Cache extends Cache[String] - def pomSha1: String = pomSha1Cache{ resolveHash("pom", false) } - private object jarCache extends Cache[File] - def jar: File = jarCache{ resolve("jar", Some(jarSha1), true) } - private object pomCache extends Cache[File] - def pom: File = pomCache{ resolve("pom", Some(pomSha1), false) } + def jarSha1: String = taskCache[BoundMavenDependency]("jarSha1").memoize{ resolveHash("jar", true) } + def pomSha1: String = taskCache[BoundMavenDependency]("pomSha1").memoize{ resolveHash("pom", false) } + def jar: File = taskCache[BoundMavenDependency]("jar").memoize{ resolve("jar", Some(jarSha1), true) } + def pom: File = taskCache[BoundMavenDependency]("pom").memoize{ resolve("pom", Some(pomSha1), false) } private def pomXml = XML.loadFile(pom.string) // ========== pom traversal ========== @@ -278,7 +284,7 @@ case class BoundMavenDependency( (parent \ "version").text ), repositories - )(logger) + ) }.flatMap(_.transitivePom) :+ this } diff --git a/stage2/BasicBuild.scala b/stage2/BasicBuild.scala index 8b4a3a5..2fd34c7 100644 --- a/stage2/BasicBuild.scala +++ b/stage2/BasicBuild.scala @@ -9,6 +9,8 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with Trigge // will create new instances given the context, which means operations in the // overrides will happen multiple times and if they are not idempotent stuff likely breaks def context: Context + def moduleKey: String = "BaseBuild("+projectDirectory.string+")" + implicit def transientCache: java.util.Map[AnyRef,AnyRef] = context.transientCache // library available to builds implicit protected final val logger: Logger = context.logger @@ -59,7 +61,7 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with Trigge // FIXME: this should probably be removed Resolver( mavenCentral ).bind( "org.scala-lang" % "scala-library" % scalaVersion - ) :+ BinaryDependency(localJars, Nil) + ) ++ ( if(localJars.nonEmpty) Seq( BinaryDependency(localJars, Nil) ) else Nil ) // ========== paths ========== final private val defaultSourceDirectory = projectDirectory ++ "/src" @@ -135,15 +137,13 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with Trigge "-unchecked" ) - private object needsUpdateCache extends Cache[Boolean] - def needsUpdate: Boolean = needsUpdateCache( + def needsUpdate: Boolean = taskCache[BaseBuild]("needsUpdate").memoize[java.lang.Boolean]( context.cbtHasChanged || lib.needsUpdate( sourceFiles, compileStatusFile ) || transitiveDependencies.filterNot(_ == context.parentBuild).exists(_.needsUpdate) ) - private object compileCache extends Cache[Option[File]] - def compile: Option[File] = compileCache{ + def compile: Option[File] = taskCache[BaseBuild]("compile").memoize{ lib.compile( context.cbtHasChanged, needsUpdate || context.parentBuild.map(_.needsUpdate).getOrElse(false), @@ -153,7 +153,6 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with Trigge ) } - def mainClasses: Seq[Class[_]] = compile.toSeq.flatMap( lib.mainClasses( _, classLoader(classLoaderCache) ) ) def runClass: Option[String] = lib.runClass( mainClasses ).map( _.getName ) @@ -264,24 +263,6 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with Trigge def finalBuild: BuildInterface = this override def show = this.getClass.getSimpleName ++ "(" ++ projectDirectory.string ++ ")" - // TODO: allow people not provide the method name, maybe via macro - // TODO: pull this out into lib - /** - caches given value in context keyed with given key and projectDirectory - the context is fresh on every complete run of cbt - */ - def cached[T <: AnyRef](name: String)(task: => T): T = { - val cache = context.transientCache - val key = (projectDirectory,name) - if( cache.containsKey(key) ){ - cache.get(key).asInstanceOf[T] - } else{ - val value = task - cache.put( key, value ) - value - } - } - // a method that can be called only to trigger any side-effects final def `void` = () } diff --git a/stage2/BuildBuild.scala b/stage2/BuildBuild.scala index cf515bb..1b05214 100644 --- a/stage2/BuildBuild.scala +++ b/stage2/BuildBuild.scala @@ -32,8 +32,7 @@ trait BuildBuildWithoutEssentials extends BaseBuild{ override def dependencies = super.dependencies :+ context.cbtDependency def managedBuildDirectory: java.io.File = lib.realpath( projectDirectory.parent ) - private object managedBuildCache extends Cache[BuildInterface] - def managedBuild = managedBuildCache{ + def managedBuild = taskCache[BuildBuildWithoutEssentials]("managedBuild").memoize{ val managedBuildFile = projectDirectory++"/build.scala" logger.composition("Loading build at " ++ managedBuildDirectory.toString) val build = ( diff --git a/stage2/BuildDependency.scala b/stage2/BuildDependency.scala index 197a7a1..4b4fdc1 100644 --- a/stage2/BuildDependency.scala +++ b/stage2/BuildDependency.scala @@ -16,9 +16,12 @@ trait TriggerLoop extends DependencyImplementation{ } /** You likely want to use the factory method in the BasicBuild class instead of this. */ final case class DirectoryDependency(context: Context) extends TriggerLoop{ + override def toString = show override def show = this.getClass.getSimpleName ++ "(" ++ context.projectDirectory.string ++ ")" + def moduleKey = this.getClass.getName ++ "("+context.projectDirectory.string+")" lazy val logger = context.logger override lazy val lib: Lib = new Lib(logger) + def transientCache = context.transientCache private lazy val root = lib.loadRoot( context.copy(args=Seq()) ) lazy val build = root.finalBuild def exportedClasspath = ClassPath() diff --git a/stage2/GitDependency.scala b/stage2/GitDependency.scala index 650fd09..e27eff9 100644 --- a/stage2/GitDependency.scala +++ b/stage2/GitDependency.scala @@ -15,15 +15,14 @@ case class GitDependency( )(implicit val logger: Logger, classLoaderCache: ClassLoaderCache, context: Context ) extends DependencyImplementation{ import GitDependency._ override def lib = new Lib(logger) - + def moduleKey = this.getClass.getName ++ "(" ++ url ++ subDirectory.map("/" ++ _).getOrElse("") ++ "#" ++ ref ++ ")" + def transientCache = context.transientCache // TODO: add support for authentication via ssh and/or https // See http://www.codeaffine.com/2014/12/09/jgit-authentication/ private val GitUrl( _, domain, path ) = url private val credentialsFile = context.projectDirectory ++ "/git.login" - private object checkoutCache extends Cache[File] - private def authenticate(_git: CloneCommand) = if(!credentialsFile.exists){ _git @@ -36,7 +35,7 @@ case class GitDependency( _git.setCredentialsProvider( new UsernamePasswordCredentialsProvider(user, password) ) } - def checkout: File = checkoutCache{ + def checkout: File = taskCache[GitDependency]("checkout").memoize{ val checkoutDirectory = context.cache ++ s"/git/$domain/$path/$ref" val _git = if(checkoutDirectory.exists){ logger.git(s"Found existing checkout of $url#$ref in $checkoutDirectory") @@ -65,8 +64,7 @@ case class GitDependency( assert( actualRef == ref, s"actual ref '$actualRef' does not match expected ref '$ref'") checkoutDirectory } - private object dependencyCache extends Cache[DependencyImplementation] - def dependency = dependencyCache{ + def dependency = taskCache[GitDependency]("dependency").memoize{ DirectoryDependency( context.copy( projectDirectory = checkout ++ subDirectory.map("/" ++ _).getOrElse("") diff --git a/stage2/Lib.scala b/stage2/Lib.scala index 769cd97..c570ca3 100644 --- a/stage2/Lib.scala +++ b/stage2/Lib.scala @@ -86,7 +86,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ compileArgs: Seq[String], classLoaderCache: ClassLoaderCache, mavenCache: File - ): Option[File] = { + )(implicit transientCache: java.util.Map[AnyRef,AnyRef]): Option[File] = { if(sourceFiles.isEmpty){ None } else { @@ -101,7 +101,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ runMain( "scala.tools.nsc.ScalaDoc", args, - ScalaDependencies(cbtHasChanged,mavenCache,scalaVersion)(logger).classLoader(classLoaderCache) + new ScalaDependencies(cbtHasChanged,mavenCache,scalaVersion).classLoader(classLoaderCache) ) } lib.jarFile( diff --git a/stage2/PackageJars.scala b/stage2/PackageJars.scala index ff89284..3ecceb2 100644 --- a/stage2/PackageJars.scala +++ b/stage2/PackageJars.scala @@ -10,18 +10,15 @@ trait PackageJars extends BaseBuild with ArtifactInfo{ Seq(() => jar, () => docJar, () => srcJar) )( _() ).flatten - private object cacheJarBasicBuild extends Cache[Option[File]] - def jar: Option[File] = cacheJarBasicBuild{ + def jar: Option[File] = taskCache[PackageJars]("jar").memoize{ compile.flatMap( lib.jar( artifactId, scalaMajorVersion, version, _, jarTarget ) ) } - private object cacheSrcJarBasicBuild extends Cache[Option[File]] - def srcJar: Option[File] = cacheSrcJarBasicBuild{ + def srcJar: Option[File] = taskCache[PackageJars]("srcJar").memoize{ lib.srcJar( sourceFiles, artifactId, scalaMajorVersion, version, scalaTarget ) } - private object cacheDocBasicBuild extends Cache[Option[File]] - def docJar: Option[File] = cacheDocBasicBuild{ + def docJar: Option[File] = taskCache[PackageJars]("docJar").memoize{ lib.docJar( context.cbtHasChanged, scalaVersion, sourceFiles, compileClasspath, docTarget, diff --git a/stage2/Stage2.scala b/stage2/Stage2.scala index ab7b4fe..2884ddb 100644 --- a/stage2/Stage2.scala +++ b/stage2/Stage2.scala @@ -24,7 +24,7 @@ object Stage2 extends Stage2Base{ } val task = args.args.lift( taskIndex ) - val context: Context = ContextImplementation( + val context: Context = new ContextImplementation( args.cwd, args.cwd, args.args.drop( taskIndex +1 ).toArray, @@ -33,7 +33,7 @@ object Stage2 extends Stage2Base{ args.cbtHasChanged, null, args.persistentCache, - new HashMap, + args.transientCache, args.cache, args.cbtHome, args.cbtHome, diff --git a/stage2/ToolsTasks.scala b/stage2/ToolsTasks.scala index b92cb7a..839780a 100644 --- a/stage2/ToolsTasks.scala +++ b/stage2/ToolsTasks.scala @@ -15,6 +15,7 @@ class ToolsTasks( import paths._ private def Resolver( urls: URL* ) = MavenResolver(cbtHasChanged,mavenCache,urls: _*) implicit val logger: Logger = lib.logger + implicit val transientCache: java.util.Map[AnyRef,AnyRef] = new java.util.HashMap def createMain: Unit = lib.createMain( cwd ) def createBuild: Unit = lib.createBuild( cwd ) def gui = NailgunLauncher.main(Array( diff --git a/stage2/plugins/Dotty.scala b/stage2/plugins/Dotty.scala index 8671fb6..fe949a3 100644 --- a/stage2/plugins/Dotty.scala +++ b/stage2/plugins/Dotty.scala @@ -14,8 +14,7 @@ trait Dotty extends BaseBuild{ context.classLoaderCache, dottyVersion = dottyVersion ) - private object compileCache extends Cache[Option[File]] - override def compile: Option[File] = compileCache{ + override def compile: Option[File] = taskCache[Dotty]("compile").memoize{ dottyLib.compile( needsUpdate || context.parentBuild.map(_.needsUpdate).getOrElse(false), sourceFiles, compileTarget, compileStatusFile, compileClasspath, @@ -41,7 +40,7 @@ class DottyLib( mavenCache: File, classLoaderCache: ClassLoaderCache, dottyVersion: String -){ +)(implicit transientCache: java.util.Map[AnyRef,AnyRef]){ val lib = new Lib(logger) import lib._ diff --git a/test/test.scala b/test/test.scala index ae6c301..332b61e 100644 --- a/test/test.scala +++ b/test/test.scala @@ -104,10 +104,11 @@ object Main{ val cache = cbtHome ++ "/cache" val mavenCache = cache ++ "/maven" val cbtHasChanged = true - def Resolver(urls: URL*) = MavenResolver(cbtHasChanged, mavenCache, urls: _*) + implicit val transientCache: java.util.Map[AnyRef,AnyRef] = new java.util.HashMap + def Resolver(urls: URL*) = MavenResolver(cbtLastModified, mavenCache, urls: _*) { - val noContext = ContextImplementation( + val noContext = new ContextImplementation( cbtHome ++ "/test/nothing", cbtHome, Array(), -- cgit v1.2.3 From e8673866b79f7473391dcee26243eee80d5d3cb6 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Thu, 9 Feb 2017 21:20:11 -0500 Subject: idempotent change propagation using lastModified instead of a non-idempotent needsUpdate flag this fixes a bug where dependees would not be rebuilt if cbt exited or was killed after dependencies were already rebuilt. --- circle.yml | 2 +- compatibility/Context.java | 32 ++++- compatibility/Dependency.java | 16 ++- compatibility/IncompatibleCbtVersionException.java | 10 ++ libraries/eval/build/build.scala | 2 +- nailgun_launcher/BuildStage1Result.java | 8 +- nailgun_launcher/CbtURLClassLoader.java | 8 +- nailgun_launcher/ClassLoaderCache.java | 66 ++++++++++ nailgun_launcher/EarlyDependencies.java | 140 +++++++++++---------- nailgun_launcher/JavaCache.java | 40 ------ nailgun_launcher/NailgunLauncher.java | 80 +++++++----- nailgun_launcher/Stage0Lib.java | 53 ++++++-- plugins/scalajs/ScalaJsBuild.scala | 2 +- plugins/scalajs/ScalaJsLib.scala | 4 +- plugins/scalatest/ScalaTest.scala | 2 +- plugins/wartremover/WartRemover.scala | 2 +- stage1/CbtPaths.scala | 2 + stage1/ClassLoaderCache.scala | 24 ---- stage1/ContextImplementation.scala | 40 +++--- stage1/MavenRepository.scala | 6 +- stage1/Stage1.scala | 100 +++++++-------- stage1/Stage1Lib.scala | 81 ++++++------ stage1/cbt.scala | 23 ++-- stage1/resolver.scala | 102 +++++++-------- stage2/BasicBuild.scala | 21 ++-- stage2/BuildDependency.scala | 2 +- stage2/GitDependency.scala | 2 +- stage2/Lib.scala | 6 +- stage2/PackageJars.scala | 4 +- stage2/Stage2.scala | 4 +- stage2/ToolsStage2.scala | 2 +- stage2/ToolsTasks.scala | 26 ++-- stage2/plugins/Dotty.scala | 44 +++---- test/test.scala | 4 +- 34 files changed, 544 insertions(+), 416 deletions(-) create mode 100644 compatibility/IncompatibleCbtVersionException.java create mode 100644 nailgun_launcher/ClassLoaderCache.java delete mode 100644 nailgun_launcher/JavaCache.java delete mode 100644 stage1/ClassLoaderCache.scala (limited to 'stage2/BasicBuild.scala') diff --git a/circle.yml b/circle.yml index 1615ad5..9de5b42 100644 --- a/circle.yml +++ b/circle.yml @@ -1,6 +1,6 @@ machine: java: - version: oraclejdk7 + version: oraclejdk8 dependencies: cache_directories: diff --git a/compatibility/Context.java b/compatibility/Context.java index a7af740..afd0b15 100644 --- a/compatibility/Context.java +++ b/compatibility/Context.java @@ -1,21 +1,43 @@ package cbt; import java.io.*; import java.util.*; +import java.util.concurrent.*; // TODO: try to reduce the number of members -public abstract class Context{ +public interface Context{ + // recently added methods that needs default values for old versions to work + public default long cbtLastModified(){ + throw new IncompatibleCbtVersionException("You need to define method cbtLastModified."); + }; + public default Map persistentCache(){ + throw new IncompatibleCbtVersionException("You need to define method persistentCache."); + }; + public default Map transientCache(){ + throw new IncompatibleCbtVersionException("You need to define method transientCache."); + }; + public default long start(){ + throw new IncompatibleCbtVersionException("You need to define method start."); + }; + + // methods that exist for longer which every CBT version in use should have by now, no default values needed public abstract File projectDirectory(); public abstract File cwd(); // REPLACE by something that allows to run cbt on some other directly public abstract String[] argsArray(); // replace this by https://github.com/cvogt/cbt/issues/172 ? public abstract String[] enabledLoggersArray(); - public abstract Long startCompat(); - public abstract Boolean cbtHasChangedCompat(); public abstract String scalaVersionOrNull(); // needed to propagate scalaVersion to dependendee builds - public abstract Map persistentCache(); - public abstract Map transientCache(); public abstract File cache(); public abstract File cbtHome(); public abstract File cbtRootHome(); // REMOVE public abstract File compatibilityTarget(); // maybe replace this with search in the classloader for it? public abstract BuildInterface parentBuildOrNull(); + + // deprecated methods + @java.lang.Deprecated + public abstract Long startCompat(); + @java.lang.Deprecated + public abstract Boolean cbtHasChangedCompat(); + @java.lang.Deprecated + public abstract ConcurrentHashMap permanentKeys(); + @java.lang.Deprecated + public abstract ConcurrentHashMap permanentClassLoaders(); } diff --git a/compatibility/Dependency.java b/compatibility/Dependency.java index efb9214..1f719c2 100644 --- a/compatibility/Dependency.java +++ b/compatibility/Dependency.java @@ -2,10 +2,22 @@ package cbt; import java.io.*; public interface Dependency{ + // recently added methods that needs default values for old versions to work + public default String moduleKey(){ + throw new IncompatibleCbtVersionException("You need to define method moduleKey."); + }; + public default long lastModified(){ + throw new IncompatibleCbtVersionException("You need to define method lastModified."); + }; + + // methods that exist for longer which every CBT version in use should have by now, no default values needed public abstract String show(); - public abstract String moduleKey(); - public abstract Boolean needsUpdateCompat(); public abstract Dependency[] dependenciesArray(); public abstract File[] dependencyClasspathArray(); public abstract File[] exportedClasspathArray(); + + // deprecated methods + @java.lang.Deprecated + public abstract boolean needsUpdateCompat(); } + diff --git a/compatibility/IncompatibleCbtVersionException.java b/compatibility/IncompatibleCbtVersionException.java new file mode 100644 index 0000000..dee50fb --- /dev/null +++ b/compatibility/IncompatibleCbtVersionException.java @@ -0,0 +1,10 @@ +package cbt; + +public class IncompatibleCbtVersionException extends RuntimeException{ + public IncompatibleCbtVersionException( String msg, Throwable parent ){ + super( msg, parent ); + } + public IncompatibleCbtVersionException( String msg ){ + super( msg ); + } +} diff --git a/libraries/eval/build/build.scala b/libraries/eval/build/build.scala index 8dcaabd..7135d3f 100644 --- a/libraries/eval/build/build.scala +++ b/libraries/eval/build/build.scala @@ -2,7 +2,7 @@ import cbt._ class Build(val context: Context) extends BaseBuild{ outer => override def dependencies = super.dependencies :+ - new ScalaCompilerDependency( context.cbtHasChanged, context.paths.mavenCache, scalaVersion ) + new ScalaCompilerDependency( context.cbtLastModified, context.paths.mavenCache, scalaVersion ) override def test: Option[ExitCode] = Some{ new BasicBuild(context.copy(projectDirectory = projectDirectory ++ "/test")) with ScalaTest{ diff --git a/nailgun_launcher/BuildStage1Result.java b/nailgun_launcher/BuildStage1Result.java index 312871d..64a660d 100644 --- a/nailgun_launcher/BuildStage1Result.java +++ b/nailgun_launcher/BuildStage1Result.java @@ -1,12 +1,14 @@ package cbt; public class BuildStage1Result{ - public Boolean changed; + public long start; + public long stage1LastModified; public ClassLoader classLoader; public String stage1Classpath; public String nailgunClasspath; public String compatibilityClasspath; - public BuildStage1Result( Boolean changed, ClassLoader classLoader, String stage1Classpath, String nailgunClasspath, String compatibilityClasspath ){ - this.changed = changed; + public BuildStage1Result( long start, long stage1LastModified, ClassLoader classLoader, String stage1Classpath, String nailgunClasspath, String compatibilityClasspath ){ + this.start = start; + this.stage1LastModified = stage1LastModified; this.classLoader = classLoader; this.stage1Classpath = stage1Classpath; this.nailgunClasspath = nailgunClasspath; diff --git a/nailgun_launcher/CbtURLClassLoader.java b/nailgun_launcher/CbtURLClassLoader.java index 43d07f4..fac7050 100644 --- a/nailgun_launcher/CbtURLClassLoader.java +++ b/nailgun_launcher/CbtURLClassLoader.java @@ -15,7 +15,7 @@ public class CbtURLClassLoader extends java.net.URLClassLoader{ + "\n)" ); } - JavaCache cache = new JavaCache( new ConcurrentHashMap() ); + ConcurrentHashMap cache = new ConcurrentHashMap(); public Class loadClass(String name) throws ClassNotFoundException{ Class _class = super.loadClass(name); if(_class == null) throw new ClassNotFoundException(name); @@ -24,11 +24,11 @@ public class CbtURLClassLoader extends java.net.URLClassLoader{ public Class loadClass(String name, Boolean resolve) throws ClassNotFoundException{ //System.out.println("loadClass("+name+") on \n"+this); synchronized( cache ){ - if(!cache.contains(name)) + if(!cache.containsKey(name)) try{ - cache.put(super.loadClass(name, resolve), name); + cache.put(name, super.loadClass(name, resolve)); } catch (ClassNotFoundException e){ - cache.put(Object.class, name); + cache.put(name, Object.class); } Class _class = cache.get(name); if(_class == Object.class){ diff --git a/nailgun_launcher/ClassLoaderCache.java b/nailgun_launcher/ClassLoaderCache.java new file mode 100644 index 0000000..6bffad0 --- /dev/null +++ b/nailgun_launcher/ClassLoaderCache.java @@ -0,0 +1,66 @@ +package cbt; + +import java.util.*; +import static java.io.File.pathSeparator; +import static cbt.Stage0Lib.*; + +final public class ClassLoaderCache{ + public Map hashMap; + final ThreadLocal> seen = new ThreadLocal>(){ + @Override protected HashSet initialValue(){ + return new HashSet(); + } + }; + + public ClassLoaderCache( + Map hashMap + ){ + this.hashMap = hashMap; + } + + public ClassLoader get( String key, long timestamp ){ + seen.get().add( key ); + @SuppressWarnings("unchecked") + ClassLoader t = (ClassLoader) hashMap.get( + hashMap.get( key ) + ); + assert hashMap.get(t).equals(timestamp); + return t; + } + + public boolean containsKey( String key, long timestamp ){ + boolean contains = hashMap.containsKey( key ); + if( contains ){ + Object keyObject = hashMap.get( key ); + Object classLoader = hashMap.get( keyObject ); + long oldTimestamp = (long) hashMap.get( classLoader ); + boolean res = oldTimestamp == timestamp; + return res; + } else { + return false; + } + } + + public void put( String key, ClassLoader value, long timestamp ){ + assert !seen.get().contains( key ): "Thread tries to update cache key after observing it: " + key; + LockableJavaKey keyObject = new LockableJavaKey(); + hashMap.put( key, keyObject ); + hashMap.put( keyObject, value ); + hashMap.put( value, timestamp ); + } + + @Override public String toString(){ + StringBuilder res = new StringBuilder(); + res.append("ClassLoaderCache(\n\n"); + for( Object key: hashMap.keySet() ){ + if( key instanceof String ) + res.append( + join( "\n", key.toString().split(":") ) + " -> " + hashMap.get( hashMap.get(key) ) + + "\n\n" + ); + } + res.append("\n\n"); + return res.toString(); + } +} +class LockableJavaKey{} diff --git a/nailgun_launcher/EarlyDependencies.java b/nailgun_launcher/EarlyDependencies.java index fdb54b5..8709e69 100644 --- a/nailgun_launcher/EarlyDependencies.java +++ b/nailgun_launcher/EarlyDependencies.java @@ -4,6 +4,7 @@ import java.io.*; import java.nio.file.*; import java.net.*; import java.security.*; +import java.util.*; import static cbt.Stage0Lib.*; import static cbt.NailgunLauncher.*; @@ -28,7 +29,7 @@ class EarlyDependencies{ String scalaLibrary_2_10_6_File; public EarlyDependencies( - String mavenCache, String mavenUrl, JavaCache classLoaderCache, ClassLoader rootClassLoader + String mavenCache, String mavenUrl, ClassLoaderCache classLoaderCache, ClassLoader rootClassLoader ) throws Throwable { scalaReflect_2_11_8_File = mavenCache + "/org/scala-lang/scala-reflect/2.11.8/scala-reflect-2.11.8.jar"; scalaCompiler_2_11_8_File = mavenCache + "/org/scala-lang/scala-compiler/2.11.8/scala-compiler-2.11.8.jar"; @@ -46,95 +47,104 @@ class EarlyDependencies{ download(new URL(mavenUrl + "/org/scala-lang/scala-compiler/2.11.8/scala-compiler-2.11.8.jar"), Paths.get(scalaCompiler_2_11_8_File), "fe1285c9f7b58954c5ef6d80b59063569c065e9a"); // org.scala-lang:scala-library:2.10.6 - download(new URL(mavenUrl + "/org/scala-lang/scala-library/2.10.6/scala-library-2.10.6.jar"), Paths.get(scalaLibrary_2_10_6_File), "421989aa8f95a05a4f894630aad96b8c7b828732"); - String[] scalaLibrary_2_10_6_ClasspathArray = new String[]{scalaLibrary_2_10_6_File}; - String scalaLibrary_2_10_6_Classpath = classpath( scalaLibrary_2_10_6_ClasspathArray ); - ClassLoader scalaLibrary_2_10_6_ = - classLoaderCache.contains( scalaLibrary_2_10_6_Classpath ) - ? classLoaderCache.get( scalaLibrary_2_10_6_Classpath ) - : classLoaderCache.put( classLoader( scalaLibrary_2_10_6_File, rootClassLoader ), scalaLibrary_2_10_6_Classpath ); + ClassLoader scalaLibrary_2_10_6_ = loadDependency( + mavenUrl + "/org/scala-lang/scala-library/2.10.6/scala-library-2.10.6.jar", + scalaLibrary_2_10_6_File, + "421989aa8f95a05a4f894630aad96b8c7b828732", + classLoaderCache, + rootClassLoader, + scalaLibrary_2_10_6_ClasspathArray + ); // org.scala-lang:scala-reflect:2.10.6 - download(new URL(mavenUrl + "/org/scala-lang/scala-reflect/2.10.6/scala-reflect-2.10.6.jar"), Paths.get(scalaReflect_2_10_6_File), "3259f3df0f166f017ef5b2d385445808398c316c"); - String[] scalaReflect_2_10_6_ClasspathArray = new String[]{scalaLibrary_2_10_6_File, scalaReflect_2_10_6_File}; - String scalaReflect_2_10_6_Classpath = classpath( scalaReflect_2_10_6_ClasspathArray ); - ClassLoader scalaReflect_2_10_6_ = - classLoaderCache.contains( scalaReflect_2_10_6_Classpath ) - ? classLoaderCache.get( scalaReflect_2_10_6_Classpath ) - : classLoaderCache.put( classLoader( scalaReflect_2_10_6_File, scalaLibrary_2_10_6_ ), scalaReflect_2_10_6_Classpath ); + ClassLoader scalaReflect_2_10_6_ = loadDependency( + mavenUrl + "/org/scala-lang/scala-reflect/2.10.6/scala-reflect-2.10.6.jar", + scalaReflect_2_10_6_File, + "3259f3df0f166f017ef5b2d385445808398c316c", + classLoaderCache, + scalaLibrary_2_10_6_, + scalaReflect_2_10_6_ClasspathArray + ); // com.typesafe.sbt:sbt-interface:0.13.12 - download(new URL(mavenUrl + "/com/typesafe/sbt/sbt-interface/0.13.12/sbt-interface-0.13.12.jar"), Paths.get(sbtInterface_0_13_12_File), "fcc7875c02f0d4641fac0518121bd71475d3909b"); - String[] sbtInterface_0_13_12_ClasspathArray = new String[]{sbtInterface_0_13_12_File, scalaLibrary_2_10_6_File, scalaReflect_2_10_6_File}; - String sbtInterface_0_13_12_Classpath = classpath( sbtInterface_0_13_12_ClasspathArray ); - ClassLoader sbtInterface_0_13_12_ = - classLoaderCache.contains( sbtInterface_0_13_12_Classpath ) - ? classLoaderCache.get( sbtInterface_0_13_12_Classpath ) - : classLoaderCache.put( classLoader( sbtInterface_0_13_12_File, scalaReflect_2_10_6_ ), sbtInterface_0_13_12_Classpath ); + ClassLoader sbtInterface_0_13_12_ = loadDependency( + mavenUrl + "/com/typesafe/sbt/sbt-interface/0.13.12/sbt-interface-0.13.12.jar", + sbtInterface_0_13_12_File, + "fcc7875c02f0d4641fac0518121bd71475d3909b", + classLoaderCache, + scalaReflect_2_10_6_, + sbtInterface_0_13_12_ClasspathArray + ); // org.scala-lang:scala-compiler:2.10.6 - download(new URL(mavenUrl + "/org/scala-lang/scala-compiler/2.10.6/scala-compiler-2.10.6.jar"), Paths.get(scalaCompiler_2_10_6_File), "9b15174852f5b6bb1edbf303d5722286a0a54011"); - String[] scalaCompiler_2_10_6_ClasspathArray = new String[]{sbtInterface_0_13_12_File, scalaCompiler_2_10_6_File, scalaLibrary_2_10_6_File, scalaReflect_2_10_6_File}; - String scalaCompiler_2_10_6_Classpath = classpath( scalaCompiler_2_10_6_ClasspathArray ); - ClassLoader scalaCompiler_2_10_6_ = - classLoaderCache.contains( scalaCompiler_2_10_6_Classpath ) - ? classLoaderCache.get( scalaCompiler_2_10_6_Classpath ) - : classLoaderCache.put( classLoader( scalaCompiler_2_10_6_File, sbtInterface_0_13_12_ ), scalaCompiler_2_10_6_Classpath ); + ClassLoader scalaCompiler_2_10_6_ = loadDependency( + mavenUrl + "/org/scala-lang/scala-compiler/2.10.6/scala-compiler-2.10.6.jar", + scalaCompiler_2_10_6_File, + "9b15174852f5b6bb1edbf303d5722286a0a54011", + classLoaderCache, + sbtInterface_0_13_12_, + scalaCompiler_2_10_6_ClasspathArray + ); // com.typesafe.sbt:compiler-interface:0.13.12 - download(new URL(mavenUrl + "/com/typesafe/sbt/compiler-interface/0.13.12/compiler-interface-0.13.12-sources.jar"), Paths.get(compilerInterface_0_13_12_File), "d9c3270576e162bf017b146af262364c2db87a32"); - String[] compilerInterface_0_13_12_ClasspathArray = new String[]{compilerInterface_0_13_12_File, sbtInterface_0_13_12_File, scalaCompiler_2_10_6_File, scalaLibrary_2_10_6_File, scalaReflect_2_10_6_File}; - String compilerInterface_0_13_12_Classpath = classpath( compilerInterface_0_13_12_ClasspathArray ); - ClassLoader compilerInterface_0_13_12_ = - classLoaderCache.contains( compilerInterface_0_13_12_Classpath ) - ? classLoaderCache.get( compilerInterface_0_13_12_Classpath ) - : classLoaderCache.put( classLoader( compilerInterface_0_13_12_File, scalaCompiler_2_10_6_ ), compilerInterface_0_13_12_Classpath ); + ClassLoader compilerInterface_0_13_12_ = loadDependency( + mavenUrl + "/com/typesafe/sbt/compiler-interface/0.13.12/compiler-interface-0.13.12-sources.jar", + compilerInterface_0_13_12_File, + "d9c3270576e162bf017b146af262364c2db87a32", + classLoaderCache, + scalaCompiler_2_10_6_, + compilerInterface_0_13_12_ClasspathArray + ); // com.typesafe.sbt:incremental-compiler:0.13.12 - download(new URL(mavenUrl + "/com/typesafe/sbt/incremental-compiler/0.13.12/incremental-compiler-0.13.12.jar"), Paths.get(incrementalCompiler_0_13_12_File), "259f6d24a5a3791bb233787d6a8e639c4ab86fe5"); - String[] incrementalCompiler_0_13_12_ClasspathArray = new String[]{compilerInterface_0_13_12_File, incrementalCompiler_0_13_12_File, sbtInterface_0_13_12_File, scalaCompiler_2_10_6_File, scalaLibrary_2_10_6_File, scalaReflect_2_10_6_File}; - String incrementalCompiler_0_13_12_Classpath = classpath( incrementalCompiler_0_13_12_ClasspathArray ); - ClassLoader incrementalCompiler_0_13_12_ = - classLoaderCache.contains( incrementalCompiler_0_13_12_Classpath ) - ? classLoaderCache.get( incrementalCompiler_0_13_12_Classpath ) - : classLoaderCache.put( classLoader( incrementalCompiler_0_13_12_File, compilerInterface_0_13_12_ ), incrementalCompiler_0_13_12_Classpath ); + ClassLoader incrementalCompiler_0_13_12_ = loadDependency( + mavenUrl + "/com/typesafe/sbt/incremental-compiler/0.13.12/incremental-compiler-0.13.12.jar", + incrementalCompiler_0_13_12_File, + "259f6d24a5a3791bb233787d6a8e639c4ab86fe5", + classLoaderCache, + compilerInterface_0_13_12_, + incrementalCompiler_0_13_12_ClasspathArray + ); // com.typesafe.zinc:zinc:0.3.12 - download(new URL(mavenUrl + "/com/typesafe/zinc/zinc/0.3.12/zinc-0.3.12.jar"), Paths.get(zinc_0_3_12_File), "c4339e93f5b7273f49ad026248f4fdb1d4d6c7c4"); - String[] zinc_0_3_12_ClasspathArray = new String[]{compilerInterface_0_13_12_File, incrementalCompiler_0_13_12_File, sbtInterface_0_13_12_File, zinc_0_3_12_File, scalaCompiler_2_10_6_File, scalaLibrary_2_10_6_File, scalaReflect_2_10_6_File}; - String zinc_0_3_12_Classpath = classpath( zinc_0_3_12_ClasspathArray ); - ClassLoader zinc_0_3_12_ = - classLoaderCache.contains( zinc_0_3_12_Classpath ) - ? classLoaderCache.get( zinc_0_3_12_Classpath ) - : classLoaderCache.put( classLoader( zinc_0_3_12_File, incrementalCompiler_0_13_12_ ), zinc_0_3_12_Classpath ); + ClassLoader zinc_0_3_12_ = loadDependency( + mavenUrl + "/com/typesafe/zinc/zinc/0.3.12/zinc-0.3.12.jar", + zinc_0_3_12_File, + "c4339e93f5b7273f49ad026248f4fdb1d4d6c7c4", + classLoaderCache, + incrementalCompiler_0_13_12_, + zinc_0_3_12_ClasspathArray + ); // org.scala-lang:scala-library:2.11.8 - download(new URL(mavenUrl + "/org/scala-lang/scala-library/2.11.8/scala-library-2.11.8.jar"), Paths.get(scalaLibrary_2_11_8_File), "ddd5a8bced249bedd86fb4578a39b9fb71480573"); - String[] scalaLibrary_2_11_8_ClasspathArray = new String[]{scalaLibrary_2_11_8_File}; - String scalaLibrary_2_11_8_Classpath = classpath( scalaLibrary_2_11_8_ClasspathArray ); - ClassLoader scalaLibrary_2_11_8_ = - classLoaderCache.contains( scalaLibrary_2_11_8_Classpath ) - ? classLoaderCache.get( scalaLibrary_2_11_8_Classpath ) - : classLoaderCache.put( classLoader( scalaLibrary_2_11_8_File, rootClassLoader ), scalaLibrary_2_11_8_Classpath ); + ClassLoader scalaLibrary_2_11_8_ = loadDependency( + mavenUrl + "/org/scala-lang/scala-library/2.11.8/scala-library-2.11.8.jar", + scalaLibrary_2_11_8_File, + "ddd5a8bced249bedd86fb4578a39b9fb71480573", + classLoaderCache, + rootClassLoader, + scalaLibrary_2_11_8_ClasspathArray + ); // org.scala-lang.modules:scala-xml_2.11:1.0.5 - download(new URL(mavenUrl + "/org/scala-lang/modules/scala-xml_2.11/1.0.5/scala-xml_2.11-1.0.5.jar"), Paths.get(scalaXml_1_0_5_File), "77ac9be4033768cf03cc04fbd1fc5e5711de2459"); - String[] scalaXml_1_0_5_ClasspathArray = new String[]{scalaXml_1_0_5_File, scalaLibrary_2_11_8_File}; - String scalaXml_1_0_5_Classpath = classpath( scalaXml_1_0_5_ClasspathArray ); - ClassLoader scalaXml_1_0_5_ = - classLoaderCache.contains( scalaXml_1_0_5_Classpath ) - ? classLoaderCache.get( scalaXml_1_0_5_Classpath ) - : classLoaderCache.put( classLoader( scalaXml_1_0_5_File, scalaLibrary_2_11_8_ ), scalaXml_1_0_5_Classpath ); - + ClassLoader scalaXml_1_0_5_ = loadDependency( + mavenUrl + "/org/scala-lang/modules/scala-xml_2.11/1.0.5/scala-xml_2.11-1.0.5.jar", + scalaXml_1_0_5_File, + "77ac9be4033768cf03cc04fbd1fc5e5711de2459", + classLoaderCache, + scalaLibrary_2_11_8_, + scalaXml_1_0_5_ClasspathArray + ); + classLoader = scalaXml_1_0_5_; classpathArray = scalaXml_1_0_5_ClasspathArray; diff --git a/nailgun_launcher/JavaCache.java b/nailgun_launcher/JavaCache.java deleted file mode 100644 index 3ba12ab..0000000 --- a/nailgun_launcher/JavaCache.java +++ /dev/null @@ -1,40 +0,0 @@ -package cbt; - -import java.util.*; -import static java.io.File.pathSeparator; -import static cbt.Stage0Lib.*; - -final class JavaCache{ - Map hashMap; - - public JavaCache( - Map hashMap - ){ - this.hashMap = hashMap; - } - - public T get( Object key ){ - @SuppressWarnings("unchecked") - T t = (T) hashMap.get( - hashMap.get( key ) - ); - return t; - } - - public Boolean contains( Object key/*, Long timestamp*/ ){ - return hashMap.containsKey( key );/* && ( - (Long) hashMap.get( hashMap.get( hashMap.get(key) ) ) >= timestamp - );*/ - } - - public T put( Object value, Object key/*, Long timestamp*/ ){ - LockableJavaKey keyObject = new LockableJavaKey(); - hashMap.put( key, keyObject ); - hashMap.put( keyObject, value ); - //hashMap.put( value, timestamp ); - @SuppressWarnings("unchecked") - T t = (T) value; - return t; - } -} -class LockableJavaKey{} diff --git a/nailgun_launcher/NailgunLauncher.java b/nailgun_launcher/NailgunLauncher.java index c397810..8a330d8 100644 --- a/nailgun_launcher/NailgunLauncher.java +++ b/nailgun_launcher/NailgunLauncher.java @@ -26,12 +26,12 @@ public class NailgunLauncher{ @SuppressWarnings("unchecked") public static Object getBuild( Object context ) throws Throwable{ BuildStage1Result res = buildStage1( - (Boolean) get(context, "cbtHasChangedCompat"), - (Long) get(context, "startCompat"), + (long) get(context, "cbtLastModified"), + (long) get(context, "start"), ((File) get(context, "cache")).toString() + "/", ((File) get(context, "cbtHome")).toString(), ((File) get(context, "compatibilityTarget")).toString() + "/", - new JavaCache( + new ClassLoaderCache( (HashMap) get(context, "persistentCache") ) ); @@ -43,6 +43,8 @@ public class NailgunLauncher{ .invoke(null, context, res); } + public static long nailgunLauncherLastModified = -1; // this initial value should be overwritten, never read + public static void main( String[] args ) throws Throwable { Long _start = System.currentTimeMillis(); if(args[0].equals("check-alive")){ @@ -76,13 +78,18 @@ public class NailgunLauncher{ String CBT_HOME = System.getenv("CBT_HOME"); String cache = CBT_HOME + "/cache/"; String compatibilityTarget = CBT_HOME + "/compatibility/" + TARGET; - // copy cache, so that this thread has a consistent view - // replace before returning, see below - JavaCache classLoaderCache = new JavaCache( + // copy cache, so that this thread has a consistent view despite other threads + // changing their copies + // replace again before returning, see below + ClassLoaderCache classLoaderCache = new ClassLoaderCache( new HashMap(classLoaderCacheHashMap) ); + + String nailgunTarget = CBT_HOME + "/" + NAILGUN + TARGET; + long nailgunLauncherLastModified = new File( nailgunTarget + "../classes.last-success" ).lastModified(); + BuildStage1Result res = buildStage1( - false, start, cache, CBT_HOME, compatibilityTarget, classLoaderCache + nailgunLauncherLastModified, start, cache, CBT_HOME, compatibilityTarget, classLoaderCache ); try{ @@ -90,9 +97,9 @@ public class NailgunLauncher{ .classLoader .loadClass("cbt.Stage1") .getMethod( - "run", String[].class, File.class, File.class, BuildStage1Result.class, Long.class, Map.class + "run", String[].class, File.class, File.class, BuildStage1Result.class, Map.class ).invoke( - null, (Object) args, new File(cache), new File(CBT_HOME), res, start, classLoaderCache.hashMap + null, (Object) args, new File(cache), new File(CBT_HOME), res, classLoaderCache.hashMap ); System.exit( exitCode ); @@ -115,7 +122,8 @@ public class NailgunLauncher{ } public static BuildStage1Result buildStage1( - Boolean changed, long start, String cache, String cbtHome, String compatibilityTarget, JavaCache classLoaderCache + final long lastModified, final long start, final String cache, final String cbtHome, + final String compatibilityTarget, final ClassLoaderCache classLoaderCache ) throws Throwable { _assert(TARGET != null, "environment variable TARGET not defined"); String nailgunTarget = cbtHome + "/" + NAILGUN + TARGET; @@ -128,9 +136,11 @@ public class NailgunLauncher{ ClassLoader rootClassLoader = new CbtURLClassLoader( new URL[]{}, ClassLoader.getSystemClassLoader().getParent() ); // wrap for caching EarlyDependencies earlyDeps = new EarlyDependencies(mavenCache, mavenUrl, classLoaderCache, rootClassLoader); - ClassLoader compatibilityClassLoader; + long nailgunLauncherLastModified = new File( nailgunTarget + "../classes.last-success" ).lastModified(); + + long compatibilityLastModified; if(!compatibilityTarget.startsWith(cbtHome)){ - compatibilityClassLoader = classLoaderCache.get( compatibilityTarget ); + compatibilityLastModified = new File( compatibilityTarget + "../classes.last-success" ).lastModified(); } else { List compatibilitySourceFiles = new ArrayList(); for( File f: compatibilitySources.listFiles() ){ @@ -138,20 +148,20 @@ public class NailgunLauncher{ compatibilitySourceFiles.add(f); } } - changed = compile(changed, start, "", compatibilityTarget, earlyDeps, compatibilitySourceFiles); - - if( classLoaderCache.contains( compatibilityTarget ) ){ - compatibilityClassLoader = classLoaderCache.get( compatibilityTarget ); - } else { - compatibilityClassLoader = classLoaderCache.put( classLoader(compatibilityTarget, rootClassLoader), compatibilityTarget ); + + compatibilityLastModified = compile( 0L, "", compatibilityTarget, earlyDeps, compatibilitySourceFiles); + + if( !classLoaderCache.containsKey( compatibilityTarget, compatibilityLastModified ) ){ + classLoaderCache.put( compatibilityTarget, classLoader(compatibilityTarget, rootClassLoader), compatibilityLastModified ); } } + final ClassLoader compatibilityClassLoader = classLoaderCache.get( compatibilityTarget, compatibilityLastModified ); String[] nailgunClasspathArray = append( earlyDeps.classpathArray, nailgunTarget ); String nailgunClasspath = classpath( nailgunClasspathArray ); - ClassLoader nailgunClassLoader = new CbtURLClassLoader( new URL[]{}, NailgunLauncher.class.getClassLoader() ); // wrap for caching - if( !classLoaderCache.contains( nailgunClasspath ) ){ - nailgunClassLoader = classLoaderCache.put( nailgunClassLoader, nailgunClasspath ); + final ClassLoader nailgunClassLoader = new CbtURLClassLoader( new URL[]{}, NailgunLauncher.class.getClassLoader() ); // wrap for caching + if( !classLoaderCache.containsKey( nailgunClasspath, nailgunLauncherLastModified ) ){ + classLoaderCache.put( nailgunClasspath, nailgunClassLoader, nailgunLauncherLastModified ); } String[] stage1ClasspathArray = @@ -164,14 +174,24 @@ public class NailgunLauncher{ stage1SourceFiles.add(f); } } - changed = compile(changed, start, stage1Classpath, stage1Target, earlyDeps, stage1SourceFiles); - ClassLoader stage1classLoader; - if( !changed && classLoaderCache.contains( stage1Classpath ) ){ - stage1classLoader = classLoaderCache.get( stage1Classpath ); - } else { - stage1classLoader = + final long stage1BeforeCompiled = System.currentTimeMillis(); + final long stage0LastModified = Math.max( + lastModified, + Math.max( lastModified, compatibilityLastModified ) + ); + final long stage1LastModified = compile( + stage0LastModified, stage1Classpath, stage1Target, earlyDeps, stage1SourceFiles + ); + + if( stage1LastModified < compatibilityLastModified ) + throw new AssertionError( + "Cache invalidation bug: cbt compatibility layer recompiled, but cbt stage1 did not." + ); + + if( !classLoaderCache.containsKey( stage1Classpath, stage1LastModified ) ){ classLoaderCache.put( + stage1Classpath, classLoader( stage1Target, new MultiClassLoader2( @@ -180,12 +200,14 @@ public class NailgunLauncher{ earlyDeps.classLoader ) ), - stage1Classpath + stage1LastModified ); } + final ClassLoader stage1classLoader = classLoaderCache.get( stage1Classpath, stage1LastModified ); return new BuildStage1Result( - changed, + start, + stage1LastModified, stage1classLoader, stage1Classpath, nailgunClasspath, diff --git a/nailgun_launcher/Stage0Lib.java b/nailgun_launcher/Stage0Lib.java index 865b1cb..425ced3 100644 --- a/nailgun_launcher/Stage0Lib.java +++ b/nailgun_launcher/Stage0Lib.java @@ -11,6 +11,7 @@ import static java.io.File.pathSeparator; import static cbt.NailgunLauncher.*; import java.nio.file.*; import java.nio.file.attribute.FileTime; +import static java.lang.Math.min; public class Stage0Lib{ public static void _assert(Boolean condition, Object msg){ @@ -47,25 +48,57 @@ public class Stage0Lib{ return join( pathSeparator, files ); } + public static long lastModified( String... files ){ + List lastModified = new ArrayList(); + for( String file: files ){ + lastModified.add( new File(file).lastModified() ); + } + return Collections.max( lastModified ); + } + + public static ClassLoader loadDependency( + String url, + String file, + String hash, + ClassLoaderCache classLoaderCache, + ClassLoader parent, + String... classpathArray + ) throws Throwable { + download(new URL(url), Paths.get(file), hash); + + final long lastModified = lastModified( classpathArray ); + final String classpath = classpath( classpathArray ); + + if( !classLoaderCache.containsKey( classpath, lastModified ) ) + classLoaderCache.put( classpath, classLoader( file, parent ), lastModified ); + + return classLoaderCache.get( classpath, lastModified ); + } + public static File write(File file, String content, OpenOption... options) throws Throwable{ file.getParentFile().mkdirs(); Files.write(file.toPath(), content.getBytes(), options); return file; } - public static Boolean compile( - Boolean changed, Long start, String classpath, String target, + public static long compile( + long lastModified, String classpath, String target, EarlyDependencies earlyDeps, List sourceFiles ) throws Throwable{ File statusFile = new File( new File(target) + ".last-success" ); - Long lastSuccessfullCompile = statusFile.lastModified(); + long lastCompiled = statusFile.lastModified(); + + long maxLastModified = lastModified; + final long start = System.currentTimeMillis(); // <- before recursing, so we catch them all + for( File file: sourceFiles ){ - if( file.lastModified() > lastSuccessfullCompile ){ - changed = true; - break; - } + long l = file.lastModified(); + if( l > maxLastModified ) maxLastModified = l; + // performance optimization because we'll recompile and don't need to check other files + if( l > lastCompiled ) break; } - if(changed){ + + if( maxLastModified > lastCompiled ){ List zincArgs = new ArrayList( Arrays.asList( new String[]{ @@ -100,8 +133,10 @@ public class Stage0Lib{ } finally { System.setOut(oldOut); } + return statusFile.lastModified(); // can't just use `start` here as system time precision is less than milliseconds on OSX + } else { + return lastCompiled; } - return changed; } public static ClassLoader classLoader( String file ) throws Throwable{ diff --git a/plugins/scalajs/ScalaJsBuild.scala b/plugins/scalajs/ScalaJsBuild.scala index 9374f66..99f8616 100644 --- a/plugins/scalajs/ScalaJsBuild.scala +++ b/plugins/scalajs/ScalaJsBuild.scala @@ -6,7 +6,7 @@ trait ScalaJsBuild extends BaseBuild { final protected val scalaJsLib = ScalaJsLib( scalaJsVersion, scalaVersion, - context.cbtHasChanged, + context.cbtLastModified, context.classLoaderCache, context.paths.mavenCache ) diff --git a/plugins/scalajs/ScalaJsLib.scala b/plugins/scalajs/ScalaJsLib.scala index 12c1c85..f500039 100644 --- a/plugins/scalajs/ScalaJsLib.scala +++ b/plugins/scalajs/ScalaJsLib.scala @@ -3,7 +3,7 @@ import java.io.File case class ScalaJsLib( scalaJsVersion: String, scalaVersion: String, - cbtHasChanged: Boolean, classLoaderCache: ClassLoaderCache, mavenCache: File + cbtLastModified: Long, classLoaderCache: ClassLoaderCache, mavenCache: File )(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]){ sealed trait ScalaJsOutputMode { def option: String @@ -19,7 +19,7 @@ case class ScalaJsLib( } val lib = new Lib(logger) - def dep(artifactId: String) = MavenResolver( cbtHasChanged, mavenCache, mavenCentral ).bindOne( + def dep(artifactId: String) = MavenResolver( cbtLastModified, mavenCache, mavenCentral ).bindOne( MavenDependency("org.scala-js", artifactId, scalaJsVersion) ) diff --git a/plugins/scalatest/ScalaTest.scala b/plugins/scalatest/ScalaTest.scala index ee96431..5ccabc6 100644 --- a/plugins/scalatest/ScalaTest.scala +++ b/plugins/scalatest/ScalaTest.scala @@ -6,7 +6,7 @@ trait ScalaTest extends BaseBuild{ override def run: ExitCode = { import ScalaTestLib._ val _classLoader = classLoader(context.classLoaderCache) - val suiteNames = compile.map( d => discoverSuites(d, _classLoader) ).toVector.flatten + val suiteNames = compileFile.map( d => discoverSuites(d, _classLoader) ).toVector.flatten runSuites( suiteNames.map( loadSuite( _, _classLoader ) ) ) ExitCode.Success } diff --git a/plugins/wartremover/WartRemover.scala b/plugins/wartremover/WartRemover.scala index d5bbcd0..9cf8851 100644 --- a/plugins/wartremover/WartRemover.scala +++ b/plugins/wartremover/WartRemover.scala @@ -10,7 +10,7 @@ trait WartRemover extends BaseBuild { private[this] def wartremoverCompilerDependency: String = MavenResolver( - context.cbtHasChanged, + context.cbtLastModified, context.paths.mavenCache, mavenCentral).bindOne( ScalaDependency("org.wartremover", "wartremover", "1.1.1") diff --git a/stage1/CbtPaths.scala b/stage1/CbtPaths.scala index 71c2ef1..c8f2279 100644 --- a/stage1/CbtPaths.scala +++ b/stage1/CbtPaths.scala @@ -9,7 +9,9 @@ case class CbtPaths(private val cbtHome: File, private val cache: File){ private val target = NailgunLauncher.TARGET.stripSuffix("/") val stage1Target: File = stage1 ++ ("/" ++ target) val stage2Target: File = stage2 ++ ("/" ++ target) + val stage1StatusFile: File = stage1Target ++ ".last-success" val stage2StatusFile: File = stage2Target ++ ".last-success" val compatibility: File = cbtHome ++ "/compatibility" val nailgunTarget: File = nailgun ++ ("/" ++ target) + val nailgunStatusFile: File = nailgunTarget ++ ".last-success" } diff --git a/stage1/ClassLoaderCache.scala b/stage1/ClassLoaderCache.scala deleted file mode 100644 index af0970e..0000000 --- a/stage1/ClassLoaderCache.scala +++ /dev/null @@ -1,24 +0,0 @@ -package cbt - -import java.net._ -import java.util._ -import collection.JavaConverters._ - -case class ClassLoaderCache( - logger: Logger, - private[cbt] hashMap: java.util.Map[AnyRef,AnyRef] -){ - val cache = new KeyLockedLazyCache[ClassLoader]( hashMap, Some(logger) ) - override def toString = ( - s"ClassLoaderCache(" - ++ - hashMap.asScala.collect{ - case (key, value) if key.isInstanceOf[String] => - key.toString.split(":").mkString("\n") -> value - }.toVector.sortBy(_._1).map{ - case (key, value) => key + " -> " + hashMap.get(value) - }.mkString("\n\n","\n\n","\n\n") - ++ - ")" - ) -} diff --git a/stage1/ContextImplementation.scala b/stage1/ContextImplementation.scala index 30db597..69094b0 100644 --- a/stage1/ContextImplementation.scala +++ b/stage1/ContextImplementation.scala @@ -1,20 +1,28 @@ package cbt import java.io._ -import java.lang._ class ContextImplementation( - val projectDirectory: File, - val cwd: File, - val argsArray: Array[String], - val enabledLoggersArray: Array[String], - val startCompat: Long, - val cbtHasChangedCompat: Boolean, - val scalaVersionOrNull: String, - val persistentCache: java.util.Map[AnyRef,AnyRef], - val transientCache: java.util.Map[AnyRef,AnyRef], - val cache: File, - val cbtHome: File, - val cbtRootHome: File, - val compatibilityTarget: File, - val parentBuildOrNull: BuildInterface -) extends Context + override val projectDirectory: File, + override val cwd: File, + override val argsArray: Array[String], + override val enabledLoggersArray: Array[String], + override val start: Long, + override val cbtLastModified: Long, + override val scalaVersionOrNull: String, + override val persistentCache: java.util.Map[AnyRef,AnyRef], + override val transientCache: java.util.Map[AnyRef,AnyRef], + override val cache: File, + override val cbtHome: File, + override val cbtRootHome: File, + override val compatibilityTarget: File, + override val parentBuildOrNull: BuildInterface +) extends Context{ + @deprecated("this method is replaced by cbtLastModified","") + def cbtHasChangedCompat = true + @deprecated("this method is replaced by start","") + def startCompat = start + @deprecated("this methods is replaced by persistentCache","") + def permanentKeys = throw new IncompatibleCbtVersionException("You need to upgrade your CBT version in this module. The Context field permanentClassLoaders is no longer supported."); + @deprecated("this methods is replaced by persistentCache","") + def permanentClassLoaders = throw new IncompatibleCbtVersionException("You need to upgrade your CBT version in this module. The Context field permanentClassLoaders is no longer supported."); +} diff --git a/stage1/MavenRepository.scala b/stage1/MavenRepository.scala index 2ac7064..a8c9b51 100644 --- a/stage1/MavenRepository.scala +++ b/stage1/MavenRepository.scala @@ -2,12 +2,12 @@ package cbt import java.io._ import java.net._ case class MavenResolver( - cbtHasChanged: Boolean, mavenCache: File, urls: URL* + cbtLastModified: Long, mavenCache: File, urls: URL* )( implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef] ){ def bind( dependencies: MavenDependency* ): Seq[BoundMavenDependency] - = dependencies.map( BoundMavenDependency(cbtHasChanged,mavenCache,_,urls.to) ).to + = dependencies.map( BoundMavenDependency(cbtLastModified,mavenCache,_,urls.to) ).to def bindOne( dependency: MavenDependency ): BoundMavenDependency - = BoundMavenDependency( cbtHasChanged, mavenCache, dependency, urls.to ) + = BoundMavenDependency( cbtLastModified, mavenCache, dependency, urls.to ) } diff --git a/stage1/Stage1.scala b/stage1/Stage1.scala index 27c6402..1fd4663 100644 --- a/stage1/Stage1.scala +++ b/stage1/Stage1.scala @@ -39,7 +39,8 @@ abstract class Stage2Base{ case class Stage2Args( cwd: File, args: Seq[String], - cbtHasChanged: Boolean, + stage2LastModified: Long, + logger: Logger, classLoaderCache: ClassLoaderCache, cache: File, cbtHome: File, @@ -47,8 +48,9 @@ case class Stage2Args( )( implicit val transientCache: java.util.Map[AnyRef,AnyRef] ){ - val ClassLoaderCache( logger, persistentCache ) = classLoaderCache + val persistentCache = classLoaderCache.hashMap } + object Stage1{ protected def newerThan( a: File, b: File ) ={ a.lastModified > b.lastModified @@ -56,13 +58,14 @@ object Stage1{ def getBuild( _context: java.lang.Object, buildStage1: BuildStage1Result ) = { val context = _context.asInstanceOf[Context] - val logger = new Logger( context.enabledLoggers, context.start ) - val (changed, classLoader) = buildStage2( + val logger = new Logger( context.enabledLoggers, buildStage1.start ) + val (cbtLastModified, classLoader) = buildStage2( buildStage1, - ClassLoaderCache( logger, context.persistentCache ), + new ClassLoaderCache( context.persistentCache ), context.cbtHome, - context.cache - )( context.transientCache ) + context.cache, + logger + )(context.transientCache) classLoader .loadClass("cbt.Stage2") @@ -70,15 +73,16 @@ object Stage1{ .invoke( null, context.copy( - cbtHasChanged = context.cbtHasChanged || buildStage1.changed || changed // might be redundant + cbtLastModified = Math.max( context.cbtLastModified, cbtLastModified ) ) ) } def buildStage2( - buildStage1: BuildStage1Result, classLoaderCache: ClassLoaderCache, cbtHome: File, cache: File - )(implicit transientCache: java.util.Map[AnyRef,AnyRef]): (Boolean, ClassLoader) = { - import classLoaderCache.logger + buildStage1: BuildStage1Result, classLoaderCache: ClassLoaderCache, cbtHome: File, cache: File, logger: Logger + )(implicit transientCache: java.util.Map[AnyRef,AnyRef]): (Long, ClassLoader) = { + + import buildStage1._ val lib = new Stage1Lib(logger) import lib._ @@ -89,61 +93,58 @@ object Stage1{ stage2.listFiles ++ (stage2 ++ "/plugins").listFiles ).toVector.filter(_.isFile).filter(_.toString.endsWith(".scala")) - val cbtHasChanged = buildStage1.changed || lib.needsUpdate(stage2sourceFiles, stage2StatusFile) - val cls = this.getClass.getClassLoader.loadClass("cbt.NailgunLauncher") - val cbtDependency = new CbtDependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, stage2Target, new File(buildStage1.compatibilityClasspath)) + def cbtDependencies = new CbtDependencies( + mavenCache, nailgunTarget, stage1Target, stage2Target, + new File(buildStage1.compatibilityClasspath) + ) logger.stage1("Compiling stage2 if necessary") - compile( - cbtHasChanged, - cbtHasChanged, + val Some( stage2LastModified ) = compile( + buildStage1.stage1LastModified, stage2sourceFiles, stage2Target, stage2StatusFile, - cbtDependency.dependencies, + cbtDependencies.stage2Dependency.dependencies, mavenCache, Seq("-deprecation","-feature","-unchecked"), classLoaderCache, zincVersion = constants.zincVersion, scalaVersion = constants.scalaVersion )(transientCache) logger.stage1(s"calling CbtDependency.classLoader") - if( cbtHasChanged && classLoaderCache.cache.containsKey( cbtDependency.classpath.string ) ) { - classLoaderCache.cache.remove( cbtDependency.classpath.string ) - } else { - assert( - buildStage1.compatibilityClasspath === cbtDependency.stage1Dependency.compatibilityDependency.classpath.string, - "compatibility classpath different from NailgunLauncher" - ) - assert( - buildStage1.stage1Classpath === cbtDependency.stage1Dependency.classpath.string, - "stage1 classpath different from NailgunLauncher" - ) - assert( - classLoaderCache.cache.containsKey( cbtDependency.stage1Dependency.compatibilityDependency.classpath.string ), - "cbt unchanged, expected compatibility classloader to be cached" - ) - assert( - classLoaderCache.cache.containsKey( cbtDependency.stage1Dependency.classpath.string ), - "cbt unchanged, expected stage1/nailgun classloader to be cached" - ) - } - val stage2ClassLoader = cbtDependency.classLoader(classLoaderCache) + assert( + buildStage1.compatibilityClasspath === cbtDependencies.compatibilityDependency.classpath.string, + "compatibility classpath different from NailgunLauncher" + ) + assert( + buildStage1.stage1Classpath === cbtDependencies.stage1Dependency.classpath.string, + "stage1 classpath different from NailgunLauncher" + ) + assert( + classLoaderCache.containsKey( cbtDependencies.compatibilityDependency.classpath.string, cbtDependencies.compatibilityDependency.lastModified ), + "cbt unchanged, expected compatibility classloader to be cached" + ) + assert( + classLoaderCache.containsKey( cbtDependencies.stage1Dependency.classpath.string, cbtDependencies.stage1Dependency.lastModified ), + "cbt unchanged, expected stage1 classloader to be cached" + ) + + val stage2ClassLoader = cbtDependencies.stage2Dependency.classLoader(classLoaderCache) { // a few classloader sanity checks val compatibilityClassLoader = - cbtDependency.stage1Dependency.compatibilityDependency.classLoader(classLoaderCache) + cbtDependencies.compatibilityDependency.classLoader(classLoaderCache) assert( classOf[BuildInterface].getClassLoader == compatibilityClassLoader, classOf[BuildInterface].getClassLoader.toString ++ "\n\nis not the same as\n\n" ++ compatibilityClassLoader.toString ) //------------- val stage1ClassLoader = - cbtDependency.stage1Dependency.classLoader(classLoaderCache) + cbtDependencies.stage1Dependency.classLoader(classLoaderCache) assert( - classOf[Stage1Dependency].getClassLoader == stage1ClassLoader, - classOf[Stage1Dependency].getClassLoader.toString ++ "\n\nis not the same as\n\n" ++ stage1ClassLoader.toString + classOf[Stage1ArgsParser].getClassLoader == stage1ClassLoader, + classOf[Stage1ArgsParser].getClassLoader.toString ++ "\n\nis not the same as\n\n" ++ stage1ClassLoader.toString ) //------------- assert( @@ -152,7 +153,7 @@ object Stage1{ ) } - ( cbtHasChanged, stage2ClassLoader ) + ( stage2LastModified, stage2ClassLoader ) } def run( @@ -160,24 +161,23 @@ object Stage1{ cache: File, cbtHome: File, buildStage1: BuildStage1Result, - start: java.lang.Long, persistentCache: java.util.Map[AnyRef,AnyRef] ): Int = { val args = Stage1ArgsParser(_args.toVector) - val logger = new Logger(args.enabledLoggers, start) + val logger = new Logger(args.enabledLoggers, buildStage1.start) implicit val transientCache: java.util.Map[AnyRef,AnyRef] = new java.util.HashMap logger.stage1(s"Stage1 start") - val classLoaderCache = ClassLoaderCache( logger, persistentCache ) - + val classLoaderCache = new ClassLoaderCache( persistentCache ) - val (cbtHasChanged, classLoader) = buildStage2( buildStage1, classLoaderCache, cbtHome, cache ) + val (stage2LastModified, classLoader) = buildStage2( buildStage1, classLoaderCache, cbtHome, cache, logger ) val stage2Args = Stage2Args( new File( args.args(0) ), args.args.drop(1).dropWhile(_ == "direct").toVector, // launcher changes cause entire nailgun restart, so no need for them here - cbtHasChanged = cbtHasChanged, + stage2LastModified = stage2LastModified, + logger = logger, classLoaderCache = classLoaderCache, cache, cbtHome, diff --git a/stage1/Stage1Lib.scala b/stage1/Stage1Lib.scala index f0bb588..ad4b2d0 100644 --- a/stage1/Stage1Lib.scala +++ b/stage1/Stage1Lib.scala @@ -32,7 +32,7 @@ class BaseLib{ def realpath(name: File) = new File(java.nio.file.Paths.get(name.getAbsolutePath).normalize.toString) } -class Stage1Lib( val logger: Logger ) extends BaseLib{ +class Stage1Lib( logger: Logger ) extends BaseLib{ lib => implicit val implicitLogger: Logger = logger @@ -183,15 +183,10 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ } } - def needsUpdate( sourceFiles: Seq[File], statusFile: File ) = { - val lastCompile = statusFile.lastModified - sourceFiles.filter(_.lastModified > lastCompile).nonEmpty - } def compile( - cbtHasChanged: Boolean, - needsRecompile: Boolean, - files: Seq[File], + cbtLastModified: Long, + sourceFiles: Seq[File], compileTarget: File, statusFile: File, dependencies: Seq[Dependency], @@ -202,20 +197,23 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ scalaVersion: String )( implicit transientCache: java.util.Map[AnyRef, AnyRef] - ): Option[File] = { - val classpath = Dependencies(dependencies).classpath + ): Option[Long] = { + val d = Dependencies(dependencies) + val classpath = d.classpath val cp = classpath.string if(classpath.files.isEmpty) - throw new Exception("Trying to compile with empty classpath. Source files: " ++ files.toString) + throw new Exception("Trying to compile with empty classpath. Source files: " ++ sourceFiles.toString) - if( files.isEmpty ){ + if( sourceFiles.isEmpty ){ None }else{ - if( needsRecompile ){ - def Resolver(urls: URL*) = MavenResolver(cbtHasChanged, mavenCache, urls: _*) + val start = System.currentTimeMillis + val lastCompiled = statusFile.lastModified + if( d.lastModified > lastCompiled || sourceFiles.exists(_.lastModified > lastCompiled) ){ + def Resolver(urls: URL*) = MavenResolver(cbtLastModified, mavenCache, urls: _*) val zinc = Resolver(mavenCentral).bindOne(MavenDependency("com.typesafe.zinc","zinc", zincVersion)) val zincDeps = zinc.transitiveDependencies - + val sbtInterface = zincDeps .collect{ case d @ @@ -242,8 +240,6 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ val scalaReflect = Resolver(mavenCentral).bindOne(MavenDependency("org.scala-lang","scala-reflect",scalaVersion)).jar val scalaCompiler = Resolver(mavenCentral).bindOne(MavenDependency("org.scala-lang","scala-compiler",scalaVersion)).jar - val start = System.currentTimeMillis - val _class = "com.typesafe.zinc.Main" val dualArgs = Seq( @@ -264,7 +260,7 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ _class, dualArgs ++ singleArgs ++ Seq( "-cp", cp // let's put cp last. It so long - ) ++ files.map(_.toString), + ) ++ sourceFiles.map(_.toString), zinc.classLoader(classLoaderCache) ) } catch { @@ -283,7 +279,7 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ -cp \\ ${classpath.strings.mkString(":\\\n")} \\ \\ - ${files.sorted.mkString(" \\\n")} + ${sourceFiles.sorted.mkString(" \\\n")} """ ) @@ -300,8 +296,10 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ } else { System.exit(code.integer) // FIXME: let's find a better solution for error handling. Maybe a monad after all. } + Some( start ) + } else { + Some( lastCompiled ) } - Some( compileTarget ) } } def redirectOutToErr[T](code: => T): T = { @@ -360,11 +358,11 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ ) def cacheOnDisk[T] - ( cbtHasChanged: Boolean, cacheFile: File ) + ( cbtLastModified: Long, cacheFile: File ) ( deserialize: String => T ) ( serialize: T => String ) ( compute: => Seq[T] ) = { - if(!cbtHasChanged && cacheFile.exists){ + if(cacheFile.exists && cacheFile.lastModified > cbtLastModified ){ import collection.JavaConversions._ Files .readAllLines( cacheFile.toPath, StandardCharsets.UTF_8 ) @@ -380,7 +378,7 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ def dependencyTreeRecursion(root: Dependency, indent: Int = 0): String = ( ( " " * indent ) - ++ (if(root.needsUpdate) red(root.show) else root.show) + ++ root.show // (if(root.needsUpdate) red(root.show) else root.show) ++ root.dependencies.map( d => "\n" ++ dependencyTreeRecursion(d,indent + 1) ).mkString @@ -420,33 +418,40 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ def classLoaderRecursion( dependency: Dependency, latest: Map[(String,String),Dependency], cache: ClassLoaderCache ): ClassLoader = { // FIXME: shouldn't we be using KeyLockedLazyCache instead of hashmap directly here? - val d = dependency val dependencies = dependency.dependencies - def dependencyClassLoader( latest: Map[(String,String),Dependency], cache: ClassLoaderCache ): ClassLoader = { + val dependencyClassLoader: ClassLoader = { if( dependency.dependencies.isEmpty ){ // wrap for caching new cbt.URLClassLoader( ClassPath(), ClassLoader.getSystemClassLoader().getParent() ) } else if( dependencies.size == 1 ){ classLoaderRecursion( dependencies.head, latest, cache ) } else{ - val cp = d.dependencyClasspath.string - if( dependencies.exists(_.needsUpdate) && cache.cache.containsKey(cp) ){ - cache.cache.remove(cp) - } - def cl = new MultiClassLoader( dependencies.map( classLoaderRecursion(_, latest, cache) ) ) - if(d.isInstanceOf[BuildInterface]) + val lastModified = dependencies.map( _.lastModified ).max + val cp = dependency.dependencyClasspath.string + val cl = + new MultiClassLoader( + dependencies.map( classLoaderRecursion(_, latest, cache) ) + ) + if(dependency.isInstanceOf[BuildInterface]) cl // Don't cache builds right now. We need to fix invalidation first. - else - cache.cache.get( cp, cl ) + else{ + if( !cache.containsKey( cp, lastModified ) ){ + cache.put( cp, cl, lastModified ) + } + cache.get( cp, lastModified ) + } } } val a = actual( dependency, latest ) - def cl = new cbt.URLClassLoader( a.exportedClasspath, dependencyClassLoader(latest, cache) ) - if(d.isInstanceOf[BuildInterface]) - cl - else - cache.cache.get( a.classpath.string, cl ).asInstanceOf[ClassLoader] + def cl = new cbt.URLClassLoader( a.exportedClasspath, dependencyClassLoader ) + + val cp = a.classpath.string + val lastModified = a.lastModified + if( !cache.containsKey( cp, lastModified ) ){ + cache.put( cp, cl, lastModified ) + } + cache.get( cp, lastModified ) } } diff --git a/stage1/cbt.scala b/stage1/cbt.scala index 01c9303..0b0ccbf 100644 --- a/stage1/cbt.scala +++ b/stage1/cbt.scala @@ -33,6 +33,11 @@ object `package`{ def ++( s: String ): URL = new URL( url.toString ++ s ) def string = url.toString } + implicit class SeqExtensions[T](seq: Seq[T]){ + def maxOption(implicit ev: Ordering[T]): Option[T] = try{ Some(seq.max) } catch { + case e:java.lang.UnsupportedOperationException if e.getMessage === "empty.max" => None + } + } implicit class BuildInterfaceExtensions(build: BuildInterface){ import build._ // TODO: if every build has a method triggers a callback if files change @@ -52,30 +57,30 @@ object `package`{ def exportedClasspath: ClassPath = ClassPath(exportedClasspathArray.to) def classpath = exportedClasspath ++ dependencyClasspath def dependencies: Seq[Dependency] = dependenciesArray.to - def needsUpdate: Boolean = needsUpdateCompat } implicit class ContextExtensions(subject: Context){ import subject._ val paths = CbtPaths(cbtHome, cache) implicit def logger: Logger = new Logger(enabledLoggers, start) - def classLoaderCache: ClassLoaderCache = new ClassLoaderCache( logger, persistentCache ) - def cbtDependency = { + def classLoaderCache: ClassLoaderCache = new ClassLoaderCache( persistentCache ) + def cbtDependencies = { import paths._ - new CbtDependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, stage2Target, compatibilityTarget)(logger, transientCache) + new CbtDependencies(mavenCache, nailgunTarget, stage1Target, stage2Target, compatibilityTarget)(logger, transientCache) } + val cbtDependency = cbtDependencies.stage2Dependency + def args: Seq[String] = argsArray.to def enabledLoggers: Set[String] = enabledLoggersArray.to def scalaVersion = Option(scalaVersionOrNull) def parentBuild = Option(parentBuildOrNull) - def start: scala.Long = startCompat - def cbtHasChanged: scala.Boolean = cbtHasChangedCompat + def cbtLastModified: scala.Long = subject.cbtLastModified def copy( projectDirectory: File = projectDirectory, args: Seq[String] = args, //enabledLoggers: Set[String] = enabledLoggers, - cbtHasChanged: Boolean = cbtHasChanged, + cbtLastModified: Long = cbtLastModified, scalaVersion: Option[String] = scalaVersion, cbtHome: File = cbtHome, parentBuild: Option[BuildInterface] = None @@ -84,8 +89,8 @@ object `package`{ cwd, args.to, enabledLoggers.to, - startCompat, - cbtHasChangedCompat, + start, + cbtLastModified, scalaVersion.getOrElse(null), persistentCache, transientCache, diff --git a/stage1/resolver.scala b/stage1/resolver.scala index 8e46135..4a39d14 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -18,22 +18,14 @@ trait DependencyImplementation extends Dependency{ */ protected lazy val taskCache = new PerClassCache(transientCache, moduleKey) - /** - CAREFUL: this is never allowed to return true for the same dependency more than - once in a single cbt run. Otherwise we can end up with multiple classloaders - for the same classes since classLoaderRecursion recreates the classLoader when it - sees this flag and it can be called multiple times. Maybe we can find a safer - solution than this current state. - */ - def needsUpdate: Boolean - //def cacheClassLoader: Boolean = false private[cbt] def targetClasspath: ClassPath def dependencyClasspathArray: Array[File] = dependencyClasspath.files.toArray def exportedClasspathArray: Array[File] = exportedClasspath.files.toArray def exportedClasspath: ClassPath def dependenciesArray: Array[Dependency] = dependencies.to - def needsUpdateCompat: java.lang.Boolean = needsUpdate + @deprecated("this method is replaced by lastModifiedCompat","") + def needsUpdateCompat = true /* //private type BuildCache = KeyLockedLazyCache[Dependency, Future[ClassPath]] @@ -122,63 +114,63 @@ trait DependencyImplementation extends Dependency{ } // TODO: all this hard codes the scala version, needs more flexibility -class ScalaCompilerDependency(cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-compiler",version, Classifier.none), Seq(mavenCentral)) -class ScalaLibraryDependency (cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-library",version, Classifier.none), Seq(mavenCentral)) -class ScalaReflectDependency (cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-reflect",version, Classifier.none), Seq(mavenCentral)) +class ScalaCompilerDependency(cbtLastModified: Long, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BoundMavenDependency(cbtLastModified, mavenCache, MavenDependency("org.scala-lang","scala-compiler",version, Classifier.none), Seq(mavenCentral)) +class ScalaLibraryDependency (cbtLastModified: Long, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BoundMavenDependency(cbtLastModified, mavenCache, MavenDependency("org.scala-lang","scala-library",version, Classifier.none), Seq(mavenCentral)) +class ScalaReflectDependency (cbtLastModified: Long, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BoundMavenDependency(cbtLastModified, mavenCache, MavenDependency("org.scala-lang","scala-reflect",version, Classifier.none), Seq(mavenCentral)) -class ScalaDependencies(cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends Dependencies( +class ScalaDependencies(cbtLastModified: Long, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends Dependencies( Seq( - new ScalaCompilerDependency(cbtHasChanged, mavenCache, version), - new ScalaLibraryDependency(cbtHasChanged, mavenCache, version), - new ScalaReflectDependency(cbtHasChanged, mavenCache, version) + new ScalaCompilerDependency(cbtLastModified, mavenCache, version), + new ScalaLibraryDependency(cbtLastModified, mavenCache, version), + new ScalaReflectDependency(cbtLastModified, mavenCache, version) ) ) case class BinaryDependency( paths: Seq[File], dependencies: Seq[Dependency] )(implicit val logger: Logger, val transientCache: java.util.Map[AnyRef,AnyRef]) extends DependencyImplementation{ assert(paths.nonEmpty) def exportedClasspath = ClassPath(paths) - override def needsUpdate = false + override def lastModified = paths.map(_.lastModified).maxOption.getOrElse(0) // FIXME: cache this def targetClasspath = exportedClasspath def moduleKey = this.getClass.getName ++ "(" ++ paths.mkString(", ") ++ ")" } /** Allows to easily assemble a bunch of dependencies */ case class Dependencies( dependencies: Seq[Dependency] )(implicit val logger: Logger, val transientCache: java.util.Map[AnyRef,AnyRef]) extends DependencyImplementation{ - override def needsUpdate = dependencies.exists(_.needsUpdate) - override def exportedClasspath = ClassPath() - override def targetClasspath = ClassPath() + override def lastModified = dependencies.map(_.lastModified).maxOption.getOrElse(0) def moduleKey = this.getClass.getName ++ "(" ++ dependencies.map(_.moduleKey).mkString(", ") ++ ")" + def targetClasspath = ClassPath() + def exportedClasspath = ClassPath() } -class Stage1Dependency(cbtHasChanged: Boolean, mavenCache: File, nailgunTarget: File, stage1Target: File, compatibilityTarget: File)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BinaryDependency( - Seq(nailgunTarget, stage1Target), - Seq( - new CompatibilityDependency(cbtHasChanged, compatibilityTarget) - ) ++ - MavenResolver(cbtHasChanged,mavenCache,mavenCentral).bind( - MavenDependency("org.scala-lang","scala-library",constants.scalaVersion), - MavenDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,constants.scalaXmlVersion) - ) -){ - val compatibilityDependency = new CompatibilityDependency(cbtHasChanged, compatibilityTarget) - +case class PostBuildDependency(target: File, _dependencies: Seq[DependencyImplementation])(implicit val logger: Logger, val transientCache: java.util.Map[AnyRef,AnyRef]) extends DependencyImplementation{ + override final lazy val lastModified = (target++".last-success").lastModified + def moduleKey = target.string + override def targetClasspath = exportedClasspath + override def exportedClasspath = ClassPath( Seq(target) ) + override def dependencies = _dependencies } - -class CompatibilityDependency(cbtHasChanged: Boolean, compatibilityTarget: File)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BinaryDependency( - Seq(compatibilityTarget), Nil -) - -class CbtDependency(cbtHasChanged: Boolean, mavenCache: File, nailgunTarget: File, stage1Target: File, stage2Target: File, compatibilityTarget: File)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef]) extends BinaryDependency( - Seq( stage2Target ), - Seq( - new Stage1Dependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, compatibilityTarget) - ) ++ - MavenResolver(cbtHasChanged, mavenCache,mavenCentral).bind( - MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0"), - MavenDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r") +case class CbtDependencies(mavenCache: File, nailgunTarget: File, stage1Target: File, stage2Target: File, compatibilityTarget: File)(implicit logger: Logger, val transientCache: java.util.Map[AnyRef,AnyRef]){ + val compatibilityDependency = PostBuildDependency(compatibilityTarget, Nil) + val cbtLastModified = (stage2Target++".last-success").lastModified + val stage1Dependency = PostBuildDependency( + stage1Target, + Seq( + PostBuildDependency(nailgunTarget, Nil), + compatibilityDependency + ) ++ + MavenResolver(cbtLastModified,mavenCache,mavenCentral).bind( + MavenDependency("org.scala-lang","scala-library",constants.scalaVersion), + MavenDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,constants.scalaXmlVersion) + ) + ) + val stage2Dependency = PostBuildDependency( + stage2Target, + stage1Dependency +: + MavenResolver(cbtLastModified, mavenCache,mavenCentral).bind( + MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0"), + MavenDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r") + ) ) -){ - val stage1Dependency = new Stage1Dependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, compatibilityTarget) } case class Classifier(name: Option[String]) @@ -191,11 +183,11 @@ abstract class DependenciesProxy{ } class BoundMavenDependencies( - cbtHasChanged: Boolean, mavenCache: File, urls: Seq[URL], mavenDependencies: Seq[MavenDependency] + cbtLastModified: Long, mavenCache: File, urls: Seq[URL], mavenDependencies: Seq[MavenDependency] )( implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef] ) extends Dependencies( - mavenDependencies.map( BoundMavenDependency(cbtHasChanged,mavenCache,_,urls) ) + mavenDependencies.map( BoundMavenDependency(cbtLastModified,mavenCache,_,urls) ) ) case class MavenDependency( groupId: String, artifactId: String, version: String, classifier: Classifier = Classifier.none @@ -209,7 +201,7 @@ object MavenDependency{ } // FIXME: take MavenResolver instead of mavenCache and repositories separately case class BoundMavenDependency( - cbtHasChanged: Boolean, mavenCache: File, mavenDependency: MavenDependency, repositories: Seq[URL] + cbtLastModified: Long, mavenCache: File, mavenDependency: MavenDependency, repositories: Seq[URL] )( implicit val logger: Logger, val transientCache: java.util.Map[AnyRef,AnyRef] ) extends ArtifactInfo with DependencyImplementation{ @@ -233,7 +225,7 @@ case class BoundMavenDependency( ) override def show: String = this.getClass.getSimpleName ++ "(" ++ mavenDependency.serialize ++ ")" - override def needsUpdate = false + override final lazy val lastModified = classpath.strings.map(new File(_).lastModified).max private val groupPath = groupId.split("\\.").mkString("/") protected[cbt] def basePath(useClassifier: Boolean) = s"/$groupPath/$artifactId/$version/$artifactId-$version" ++ (if (useClassifier) classifier.name.map("-"++_).getOrElse("") else "") @@ -276,7 +268,7 @@ case class BoundMavenDependency( (pomXml \ "parent").collect{ case parent => BoundMavenDependency( - cbtHasChanged: Boolean, + cbtLastModified: Long, mavenCache, MavenDependency( (parent \ "groupId").text, @@ -314,7 +306,7 @@ case class BoundMavenDependency( if(classifier == Classifier.sources) Seq() else { lib.cacheOnDisk( - cbtHasChanged, mavenCache ++ basePath(true) ++ ".pom.dependencies" + cbtLastModified, mavenCache ++ basePath(true) ++ ".pom.dependencies" )( MavenDependency.deserialize )( _.serialize ){ (pomXml \ "dependencies" \ "dependency").collect{ case xml if ( (xml \ "scope").text == "" || (xml \ "scope").text == "compile" ) && (xml \ "optional").text != "true" => @@ -339,7 +331,7 @@ case class BoundMavenDependency( MavenDependency( groupId, artifactId, version, classifier ) }.toVector }.map( - BoundMavenDependency( cbtHasChanged, mavenCache, _, repositories ) + BoundMavenDependency( cbtLastModified, mavenCache, _, repositories ) ).to } } diff --git a/stage2/BasicBuild.scala b/stage2/BasicBuild.scala index 2fd34c7..ef5411a 100644 --- a/stage2/BasicBuild.scala +++ b/stage2/BasicBuild.scala @@ -98,7 +98,7 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with Trigge } logEmptySourceDirectories() - def Resolver( urls: URL* ) = MavenResolver( context.cbtHasChanged, context.paths.mavenCache, urls: _* ) + def Resolver( urls: URL* ) = MavenResolver( context.cbtLastModified, context.paths.mavenCache, urls: _* ) def ScalaDependency( groupId: String, artifactId: String, version: String, classifier: Classifier = Classifier.none, @@ -126,7 +126,7 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with Trigge val resourcesDirectory = projectDirectory ++ "/resources" ClassPath( if(resourcesDirectory.exists) Seq(resourcesDirectory) else Nil ) } - def exportedClasspath : ClassPath = ClassPath(compile.toSeq) ++ resourceClasspath + def exportedClasspath : ClassPath = ClassPath(compileFile.toSeq) ++ resourceClasspath def targetClasspath = ClassPath(Seq(compileTarget)) // ========== compile, run, test ========== @@ -137,23 +137,20 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with Trigge "-unchecked" ) - def needsUpdate: Boolean = taskCache[BaseBuild]("needsUpdate").memoize[java.lang.Boolean]( - context.cbtHasChanged - || lib.needsUpdate( sourceFiles, compileStatusFile ) - || transitiveDependencies.filterNot(_ == context.parentBuild).exists(_.needsUpdate) - ) + final def lastModified: Long = compile.getOrElse(0L) + + final def compileFile: Option[File] = compile.map(_ => compileTarget) - def compile: Option[File] = taskCache[BaseBuild]("compile").memoize{ + def compile: Option[Long] = taskCache[BaseBuild]("_compile").memoize{ lib.compile( - context.cbtHasChanged, - needsUpdate || context.parentBuild.map(_.needsUpdate).getOrElse(false), + context.cbtLastModified, sourceFiles, compileTarget, compileStatusFile, compileDependencies, context.paths.mavenCache, scalacOptions, context.classLoaderCache, zincVersion = zincVersion, scalaVersion = scalaVersion ) } - def mainClasses: Seq[Class[_]] = compile.toSeq.flatMap( lib.mainClasses( _, classLoader(classLoaderCache) ) ) + def mainClasses: Seq[Class[_]] = compileFile.toSeq.flatMap( lib.mainClasses( _, classLoader(classLoaderCache) ) ) def runClass: Option[String] = lib.runClass( mainClasses ).map( _.getName ) @@ -181,7 +178,7 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with Trigge System.setProperty(colorized, "true") } - val scalac = new ScalaCompilerDependency(context.cbtHasChanged, context.paths.mavenCache, scalaVersion) + val scalac = new ScalaCompilerDependency(context.cbtLastModified, context.paths.mavenCache, scalaVersion) lib.runMain( "scala.tools.nsc.MainGenericRunner", Seq( diff --git a/stage2/BuildDependency.scala b/stage2/BuildDependency.scala index 4b4fdc1..236f958 100644 --- a/stage2/BuildDependency.scala +++ b/stage2/BuildDependency.scala @@ -27,7 +27,7 @@ final case class DirectoryDependency(context: Context) extends TriggerLoop{ def exportedClasspath = ClassPath() def dependencies = Seq(build) def triggerLoopFiles = root.triggerLoopFiles - def needsUpdate = build.needsUpdate + def lastModified = build.lastModified def targetClasspath = ClassPath() } /* diff --git a/stage2/GitDependency.scala b/stage2/GitDependency.scala index e27eff9..059d650 100644 --- a/stage2/GitDependency.scala +++ b/stage2/GitDependency.scala @@ -76,5 +76,5 @@ case class GitDependency( def exportedClasspath = ClassPath() private[cbt] def targetClasspath = exportedClasspath - def needsUpdate: Boolean = false + def lastModified: Long = dependency.lastModified } diff --git a/stage2/Lib.scala b/stage2/Lib.scala index c570ca3..a76e281 100644 --- a/stage2/Lib.scala +++ b/stage2/Lib.scala @@ -15,7 +15,7 @@ import scala.util._ case class Developer(id: String, name: String, timezone: String, url: URL) /** Don't extend. Create your own libs :). */ -final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ +final class Lib(val logger: Logger) extends Stage1Lib(logger) with Scaffold{ lib => val buildClassName = "Build" @@ -74,7 +74,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ } def docJar( - cbtHasChanged: Boolean, + cbtLastModified: Long, scalaVersion: String, sourceFiles: Seq[File], dependencyClasspath: ClassPath, @@ -101,7 +101,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ runMain( "scala.tools.nsc.ScalaDoc", args, - new ScalaDependencies(cbtHasChanged,mavenCache,scalaVersion).classLoader(classLoaderCache) + new ScalaDependencies(cbtLastModified,mavenCache,scalaVersion).classLoader(classLoaderCache) ) } lib.jarFile( diff --git a/stage2/PackageJars.scala b/stage2/PackageJars.scala index 3ecceb2..7079786 100644 --- a/stage2/PackageJars.scala +++ b/stage2/PackageJars.scala @@ -11,7 +11,7 @@ trait PackageJars extends BaseBuild with ArtifactInfo{ )( _() ).flatten def jar: Option[File] = taskCache[PackageJars]("jar").memoize{ - compile.flatMap( lib.jar( artifactId, scalaMajorVersion, version, _, jarTarget ) ) + compileFile.flatMap( lib.jar( artifactId, scalaMajorVersion, version, _, jarTarget ) ) } def srcJar: Option[File] = taskCache[PackageJars]("srcJar").memoize{ @@ -20,7 +20,7 @@ trait PackageJars extends BaseBuild with ArtifactInfo{ def docJar: Option[File] = taskCache[PackageJars]("docJar").memoize{ lib.docJar( - context.cbtHasChanged, + context.cbtLastModified, scalaVersion, sourceFiles, compileClasspath, docTarget, jarTarget, artifactId, scalaMajorVersion, version, scalacOptions, context.classLoaderCache, context.paths.mavenCache diff --git a/stage2/Stage2.scala b/stage2/Stage2.scala index 2884ddb..542a982 100644 --- a/stage2/Stage2.scala +++ b/stage2/Stage2.scala @@ -30,9 +30,9 @@ object Stage2 extends Stage2Base{ args.args.drop( taskIndex +1 ).toArray, logger.enabledLoggers.toArray, logger.start, - args.cbtHasChanged, + args.stage2LastModified, null, - args.persistentCache, + args.classLoaderCache.hashMap, args.transientCache, args.cache, args.cbtHome, diff --git a/stage2/ToolsStage2.scala b/stage2/ToolsStage2.scala index df615fc..2b5e092 100644 --- a/stage2/ToolsStage2.scala +++ b/stage2/ToolsStage2.scala @@ -4,7 +4,7 @@ object ToolsStage2 extends Stage2Base{ def run( _args: Stage2Args ): Unit = { val args = _args.args.dropWhile(Seq("tools","direct") contains _) val lib = new Lib(_args.logger) - val toolsTasks = new ToolsTasks(lib, args, _args.cwd, _args.classLoaderCache, _args.cache, _args.cbtHome, _args.cbtHasChanged) + val toolsTasks = new ToolsTasks(lib, args, _args.cwd, _args.classLoaderCache, _args.cache, _args.cbtHome, _args.stage2LastModified) new lib.ReflectObject(toolsTasks){ def usage: String = "Available methods: " ++ lib.taskNames(toolsTasks.getClass).mkString(" ") }.callNullary(args.lift(0)) diff --git a/stage2/ToolsTasks.scala b/stage2/ToolsTasks.scala index 839780a..6acf72c 100644 --- a/stage2/ToolsTasks.scala +++ b/stage2/ToolsTasks.scala @@ -9,11 +9,11 @@ class ToolsTasks( classLoaderCache: ClassLoaderCache, cache: File, cbtHome: File, - cbtHasChanged: Boolean + cbtLastModified: Long ){ private val paths = CbtPaths(cbtHome, cache) import paths._ - private def Resolver( urls: URL* ) = MavenResolver(cbtHasChanged,mavenCache,urls: _*) + private def Resolver( urls: URL* ) = MavenResolver(cbtLastModified,mavenCache,urls: _*) implicit val logger: Logger = lib.logger implicit val transientCache: java.util.Map[AnyRef,AnyRef] = new java.util.HashMap def createMain: Unit = lib.createMain( cwd ) @@ -56,7 +56,7 @@ class ToolsTasks( } def scala = { val version = args.lift(1).getOrElse(constants.scalaVersion) - val scalac = new ScalaCompilerDependency( cbtHasChanged, mavenCache, version ) + val scalac = new ScalaCompilerDependency( cbtLastModified, mavenCache, version ) val _args = Seq("-cp", scalac.classpath.string) ++ args.drop(2) lib.runMain( "scala.tools.nsc.MainGenericRunner", _args, scalac.classLoader(classLoaderCache) @@ -97,14 +97,15 @@ class ToolsTasks( val n = valName(d) s""" // ${d.groupId}:${d.artifactId}:${d.version} - download(new URL(mavenUrl + "${d.basePath(true)}.jar"), Paths.get(${n}File), "${d.jarSha1}"); - String[] ${n}ClasspathArray = new String[]{${deps.sortBy(_.jar).map(valName(_)+"File").mkString(", ")}}; - String ${n}Classpath = classpath( ${n}ClasspathArray ); - ClassLoader $n = - classLoaderCache.contains( ${n}Classpath ) - ? classLoaderCache.get( ${n}Classpath ) - : classLoaderCache.put( classLoader( ${n}File, $parentString ), ${n}Classpath );""" + ClassLoader $n = loadDependency( + mavenUrl + "${d.basePath(true)}.jar", + ${n}File, + "${d.jarSha1}", + classLoaderCache, + $parentString, + ${n}ClasspathArray + );""" } } val assignments = codeEach(zinc) ++ codeEach(scalaXml) @@ -116,6 +117,7 @@ import java.io.*; import java.nio.file.*; import java.net.*; import java.security.*; +import java.util.*; import static cbt.Stage0Lib.*; import static cbt.NailgunLauncher.*; @@ -130,13 +132,13 @@ class EarlyDependencies{ ${files.map(d => s""" String ${valName(d)}File;""").mkString("\n")} public EarlyDependencies( - String mavenCache, String mavenUrl, JavaCache classLoaderCache, ClassLoader rootClassLoader + String mavenCache, String mavenUrl, ClassLoaderCache classLoaderCache, ClassLoader rootClassLoader ) throws Throwable { ${files.map(d => s""" ${valName(d)}File = mavenCache + "${d.basePath(true)}.jar";""").mkString("\n")} ${scalaDeps.map(d => s""" download(new URL(mavenUrl + "${d.basePath(true)}.jar"), Paths.get(${valName(d)}File), "${d.jarSha1}");""").mkString("\n")} ${assignments.mkString("\n")} - + classLoader = scalaXml_${scalaXmlVersion.replace(".","_")}_; classpathArray = scalaXml_${scalaXmlVersion.replace(".","_")}_ClasspathArray; diff --git a/stage2/plugins/Dotty.scala b/stage2/plugins/Dotty.scala index fe949a3..6fe5dd3 100644 --- a/stage2/plugins/Dotty.scala +++ b/stage2/plugins/Dotty.scala @@ -8,17 +8,15 @@ trait Dotty extends BaseBuild{ def dottyVersion: String = "0.1-20160926-ec28ea1-NIGHTLY" def dottyOptions: Seq[String] = Seq() override def scalaTarget: File = target ++ s"/dotty-$dottyVersion" - + private lazy val dottyLib = new DottyLib( - logger, context.cbtHasChanged, context.paths.mavenCache, + logger, context.cbtLastModified, context.paths.mavenCache, context.classLoaderCache, dottyVersion = dottyVersion ) - override def compile: Option[File] = taskCache[Dotty]("compile").memoize{ + override def compile: Option[Long] = taskCache[Dotty]("compile").memoize{ dottyLib.compile( - needsUpdate || context.parentBuild.map(_.needsUpdate).getOrElse(false), - sourceFiles, compileTarget, compileStatusFile, compileClasspath, - dottyOptions + sourceFiles, compileTarget, compileStatusFile, compileDependencies, dottyOptions ) } @@ -36,7 +34,7 @@ trait Dotty extends BaseBuild{ class DottyLib( logger: Logger, - cbtHasChanged: Boolean, + cbtLastModified: Long, mavenCache: File, classLoaderCache: ClassLoaderCache, dottyVersion: String @@ -44,7 +42,7 @@ class DottyLib( val lib = new Lib(logger) import lib._ - private def Resolver(urls: URL*) = MavenResolver(cbtHasChanged, mavenCache, urls: _*) + private def Resolver(urls: URL*) = MavenResolver(cbtLastModified, mavenCache, urls: _*) private lazy val dottyDependency = Resolver(mavenCentral).bindOne( MavenDependency("ch.epfl.lamp","dotty_2.11",dottyVersion) ) @@ -94,22 +92,24 @@ class DottyLib( } def compile( - needsRecompile: Boolean, - files: Seq[File], + sourceFiles: Seq[File], compileTarget: File, statusFile: File, - classpath: ClassPath, + dependencies: Seq[Dependency], dottyOptions: Seq[String] - ): Option[File] = { - + ): Option[Long] = { + val d = Dependencies(dependencies) + val classpath = d.classpath + val cp = classpath.string if(classpath.files.isEmpty) - throw new Exception("Trying to compile with empty classpath. Source files: " ++ files.toString) + throw new Exception("Trying to compile with empty classpath. Source files: " ++ sourceFiles.toString) - if( files.isEmpty ){ + if( sourceFiles.isEmpty ){ None }else{ - if( needsRecompile ){ - val start = System.currentTimeMillis + val start = System.currentTimeMillis + val lastCompiled = statusFile.lastModified + if( d.lastModified > lastCompiled || sourceFiles.exists(_.lastModified > lastCompiled) ){ val _class = "dotty.tools.dotc.Main" val dualArgs = @@ -118,7 +118,7 @@ class DottyLib( ) val singleArgs = dottyOptions.map( "-S" ++ _ ) - val code = + val code = try{ System.err.println("Compiling with Dotty to " ++ compileTarget.toString) compileTarget.mkdirs @@ -128,7 +128,7 @@ class DottyLib( dualArgs ++ singleArgs ++ Seq( "-bootclasspath", dottyDependency.classpath.string, // let's put cp last. It so long "-classpath", classpath.string // let's put cp last. It so long - ) ++ files.map(_.toString), + ) ++ sourceFiles.map(_.toString), dottyDependency.classLoader(classLoaderCache) ) } @@ -150,7 +150,7 @@ ${dottyDependency.classpath.strings.mkString(":\\\n")} \\ -classpath \\ ${classpath.strings.mkString(":\\\n")} \\ \\ -${files.sorted.mkString(" \\\n")} +${sourceFiles.sorted.mkString(" \\\n")} """ ) ExitCode.Failure @@ -164,8 +164,10 @@ ${files.sorted.mkString(" \\\n")} } else { System.exit(code.integer) // FIXME: let's find a better solution for error handling. Maybe a monad after all. } + Some( start ) + } else { + Some( lastCompiled ) } - Some( compileTarget ) } } } diff --git a/test/test.scala b/test/test.scala index 332b61e..e9da8bf 100644 --- a/test/test.scala +++ b/test/test.scala @@ -103,7 +103,7 @@ object Main{ val cache = cbtHome ++ "/cache" val mavenCache = cache ++ "/maven" - val cbtHasChanged = true + val cbtLastModified = System.currentTimeMillis implicit val transientCache: java.util.Map[AnyRef,AnyRef] = new java.util.HashMap def Resolver(urls: URL*) = MavenResolver(cbtLastModified, mavenCache, urls: _*) @@ -114,7 +114,7 @@ object Main{ Array(), Array(), start, - cbtHasChanged, + cbtLastModified, null, new HashMap[AnyRef,AnyRef], new HashMap[AnyRef,AnyRef], -- cgit v1.2.3