diff options
author | Christopher Vogt <oss.nsp@cvogt.org> | 2016-03-14 23:29:09 -0400 |
---|---|---|
committer | Christopher Vogt <oss.nsp@cvogt.org> | 2016-03-19 21:13:49 -0400 |
commit | a5f4db210aa2878eb8e15a6d9fe5235199d4aee6 (patch) | |
tree | 62ee3ddd75578f340a6c36f99188fae413585b4e | |
parent | 35a96fea9336dfcac8aac75824450cbf7dc4ae1a (diff) | |
download | cbt-a5f4db210aa2878eb8e15a6d9fe5235199d4aee6.tar.gz cbt-a5f4db210aa2878eb8e15a6d9fe5235199d4aee6.tar.bz2 cbt-a5f4db210aa2878eb8e15a6d9fe5235199d4aee6.zip |
replace two level classloader with hierarchy
replace two level classloader (one for non-cachable dependencies with a cached parent one for cachable ones) with a hierachy of classloaders corresponding
this should eventually allow re-using CBT's classloader between stage1 and stage2
this change breaks the ScalaTest support for now
-rw-r--r-- | nailgun_launcher/NailgunLauncher.java | 11 | ||||
-rw-r--r-- | stage1/ClassLoaderCache.scala | 6 | ||||
-rw-r--r-- | stage1/MultiClassLoader.scala | 26 | ||||
-rw-r--r-- | stage1/Stage1Lib.scala | 2 | ||||
-rw-r--r-- | stage1/paths.scala | 3 | ||||
-rw-r--r-- | stage1/resolver.scala | 105 | ||||
-rw-r--r-- | stage2/BuildBuild.scala | 6 | ||||
-rw-r--r-- | stage2/mixins.scala | 3 |
8 files changed, 100 insertions, 62 deletions
diff --git a/nailgun_launcher/NailgunLauncher.java b/nailgun_launcher/NailgunLauncher.java index 2278764..a0b6361 100644 --- a/nailgun_launcher/NailgunLauncher.java +++ b/nailgun_launcher/NailgunLauncher.java @@ -21,8 +21,8 @@ public class NailgunLauncher{ * Persistent cache for caching classloaders for the JVM life time. Can be used as needed by user * code to improve startup time. */ - public static ConcurrentHashMap classLoaderCache = - new ConcurrentHashMap(); + public static ConcurrentHashMap classLoaderCacheKeys = new ConcurrentHashMap(); + public static ConcurrentHashMap classLoaderCacheValues = new ConcurrentHashMap(); public static SecurityManager defaultSecurityManager = System.getSecurityManager(); @@ -47,8 +47,11 @@ public class NailgunLauncher{ newArgs[i] = args[i+2]; } - new URLClassLoader( urls ) - .loadClass(args[0]) + new URLClassLoader( urls ){ + public String toString(){ + return super.toString() + "(\n " + Arrays.toString(urls) + "\n)"; + } + }.loadClass(args[0]) .getMethod("main", String[].class) .invoke( null/* _cls.newInstance()*/, (Object) newArgs ); } diff --git a/stage1/ClassLoaderCache.scala b/stage1/ClassLoaderCache.scala index 3e77dd6..10d872d 100644 --- a/stage1/ClassLoaderCache.scala +++ b/stage1/ClassLoaderCache.scala @@ -2,11 +2,12 @@ package cbt import java.net._ import java.util.concurrent.ConcurrentHashMap +import collection.JavaConversions._ class ClassLoaderCache(logger: Logger){ val persistent = new KeyLockedLazyCache( - NailgunLauncher.classLoaderCache.asInstanceOf[ConcurrentHashMap[String,AnyRef]], - NailgunLauncher.classLoaderCache.asInstanceOf[ConcurrentHashMap[AnyRef,ClassLoader]], + NailgunLauncher.classLoaderCacheKeys.asInstanceOf[ConcurrentHashMap[String,AnyRef]], + NailgunLauncher.classLoaderCacheValues.asInstanceOf[ConcurrentHashMap[AnyRef,ClassLoader]], Some(logger) ) val transient = new KeyLockedLazyCache( @@ -14,4 +15,5 @@ class ClassLoaderCache(logger: Logger){ new ConcurrentHashMap[AnyRef,ClassLoader], Some(logger) ) + override def toString = s"""ClassLoaderCache("""+ persistent.keys.keySet.toVector.map(_.toString).sorted.map(" "++_).mkString("\n","\n","\n") +""")""" } diff --git a/stage1/MultiClassLoader.scala b/stage1/MultiClassLoader.scala index 6a0f28a..c6eb47a 100644 --- a/stage1/MultiClassLoader.scala +++ b/stage1/MultiClassLoader.scala @@ -1,23 +1,20 @@ -/* package cbt import java.net._ import scala.util.Try - import scala.collection.immutable.Seq - -class MultiClassLoader(parents: Seq[ClassLoader]) extends ClassLoader { - override def loadClass(name: String) = { - //System.err.println("LOADING CLASS "++name); - val c = parents.toStream.map{ - parent => - Try{ +class MultiClassLoader(parents: Seq[ClassLoader])(implicit val logger: Logger) extends ClassLoader with CachingClassLoader{ + override def findClass(name: String) = { + parents.find( parent => + try{ parent.loadClass(name) - }.map(Option[Class[_]](_)).recover{ - case _:ClassNotFoundException => None - }.get - }.find(_.isDefined).flatten - c.getOrElse( ClassLoader.getSystemClassLoader.loadClass(name) ) + true + } catch { + case _:ClassNotFoundException => false + } + ).map( + _.loadClass(name) + ).getOrElse( throw new ClassNotFoundException(name) ) } override def toString = ( scala.Console.BLUE @@ -31,4 +28,3 @@ class MultiClassLoader(parents: Seq[ClassLoader]) extends ClassLoader { ) ++")" ) } -*/ diff --git a/stage1/Stage1Lib.scala b/stage1/Stage1Lib.scala index 9c204ae..b3b0b23 100644 --- a/stage1/Stage1Lib.scala +++ b/stage1/Stage1Lib.scala @@ -45,6 +45,7 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ // ========== reflection ========== /** Create instance of the given class via reflection */ + /* def create(cls: String)(args: Any*)(classLoader: ClassLoader): Any = { logger.composition( logger.showInvocation("Stage1Lib.create", (cls,args,classLoader)) ) import scala.reflect.runtime.universe._ @@ -55,6 +56,7 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ val ctorm = cm.reflectConstructor( tpe.decl(termNames.CONSTRUCTOR).asMethod ) ctorm(args:_*) } + */ // ========== file system / net ========== diff --git a/stage1/paths.scala b/stage1/paths.scala index d3856c8..2006ae8 100644 --- a/stage1/paths.scala +++ b/stage1/paths.scala @@ -4,9 +4,10 @@ object paths{ val cbtHome: File = new File(Option(System.getenv("CBT_HOME")).get) val mavenCache: File = cbtHome ++ "/cache/maven" val userHome: File = new File(Option(System.getProperty("user.home")).get) + val bootstrapScala: File = cbtHome ++ "/bootstrap_scala" + val nailgun: File = new File(Option(System.getenv("NAILGUN")).get) val stage1: File = new File(Option(System.getenv("STAGE1")).get) val stage2: File = cbtHome ++ "/stage2" - val nailgun: File = new File(Option(System.getenv("NAILGUN")).get) private val target = Option(System.getenv("TARGET")).get.stripSuffix("/") val stage1Target: File = stage1 ++ ("/" ++ target) val stage2Target: File = stage2 ++ ("/" ++ target) diff --git a/stage1/resolver.scala b/stage1/resolver.scala index 87b9a24..87bab66 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -36,7 +36,6 @@ abstract class Dependency{ def jars: Seq[File] = exportedJars ++ dependencyJars def canBeCached = false - def cacheDependencyClassLoader = true //private type BuildCache = KeyLockedLazyCache[Dependency, Future[ClassPath]] def exportClasspathConcurrently: ClassPath = { @@ -86,43 +85,65 @@ abstract class Dependency{ ) } - private object classLoaderCache extends Cache[URLClassLoader] - def classLoader( classLoaderCache: ClassLoaderCache ): URLClassLoader = { - if( concurrencyEnabled ){ - // trigger concurrent building / downloading dependencies - exportClasspathConcurrently - } - val transitiveClassPath = { - (this +: transitiveDependencies).map{ - case d if d.canBeCached => Left(d) - case d => Right(d) - } - } - val buildClassPath = ClassPath.flatten( - transitiveClassPath.flatMap( - _.right.toOption.map(_.exportedClasspath) - ) - ) - val cachedClassPath = ClassPath.flatten( - transitiveClassPath.flatMap( - _.left.toOption - ).par.map(_.exportedClasspath).seq.sortBy(_.string) - ) - - if(cacheDependencyClassLoader){ - new URLClassLoader( - buildClassPath, - classLoaderCache.persistent.get( - cachedClassPath.string, - cbt.URLClassLoader( cachedClassPath, ClassLoader.getSystemClassLoader ) + def actual(current: Dependency, latest: Map[(String,String),Dependency]) = current match { + case d: ArtifactInfo => latest((d.groupId,d.artifactId)) + case d => d + } + private def dependencyClassLoader( latest: Map[(String,String),Dependency], cache: ClassLoaderCache ): ClassLoader = { + if( dependencies.isEmpty ){ + ClassLoader.getSystemClassLoader + } else if( dependencies.size == 1 ){ + dependencies.head.classLoaderRecursion( latest, cache ) + } else if( dependencies.forall(_.canBeCached) ){ + assert(transitiveDependencies.forall(_.canBeCached)) + cache.persistent.get( + dependencyClasspath.string, + new MultiClassLoader( + dependencies.map( _.classLoaderRecursion(latest, cache) ) ) ) } else { + val (cachable, nonCachable) = dependencies.partition(_.canBeCached) new URLClassLoader( - buildClassPath ++ cachedClassPath, ClassLoader.getSystemClassLoader + ClassPath.flatten( nonCachable.map(actual(_,latest)).map(_.exportedClasspath) ), + cache.persistent.get( + ClassPath.flatten( cachable.map(actual(_,latest)).map(_.exportedClasspath) ).string, + new MultiClassLoader( + cachable.map( _.classLoaderRecursion(latest, cache) ) + ) + ) + ) + new MultiClassLoader( + dependencies.map( _.classLoaderRecursion(latest, cache) ) ) } } + protected def classLoaderRecursion( latest: Map[(String,String),Dependency], cache: ClassLoaderCache ): ClassLoader = { + if( canBeCached ){ + val a = actual( this, latest ) + cache.persistent.get( + a.classpath.string, + cbt.URLClassLoader( a.exportedClasspath, dependencyClassLoader(latest, cache) ) + ) + } else { + cbt.URLClassLoader( exportedClasspath, dependencyClassLoader(latest, cache) ) + } + } + private object classLoaderCache extends Cache[ClassLoader] + def classLoader( cache: ClassLoaderCache ): ClassLoader = classLoaderCache{ + if( concurrencyEnabled ){ + // trigger concurrent building / downloading dependencies + exportClasspathConcurrently + } + classLoaderRecursion( + (this +: transitiveDependencies).collect{ + case d: ArtifactInfo => d + }.groupBy( + d => (d.groupId,d.artifactId) + ).mapValues(_.head), + cache + ) + } def classpath : ClassPath = exportedClasspath ++ dependencyClasspath def dependencyJars : Seq[File] = transitiveDependencies.flatMap(_.jars) def dependencyClasspath : ClassPath = ClassPath.flatten( transitiveDependencies.map(_.exportedClasspath) ) @@ -182,15 +203,27 @@ case class BinaryDependency( path: File, dependencies: Seq[Dependency] )(implici } case class Stage1Dependency()(implicit val logger: Logger) extends Dependency{ - def exportedClasspath = ClassPath( Seq(nailgunTarget, stage1Target) ) - def exportedJars = Seq[File]() - def dependencies = ScalaDependencies(constants.scalaVersion).dependencies def updated = false // FIXME: think this through, might allow simplifications and/or optimizations + override def canBeCached = false + private object classLoaderRecursionCache extends Cache[ClassLoader] + /* + override def classLoaderRecursion(latest: Map[(String,String),Dependency], cache: ClassLoaderCache) = classLoaderRecursionCache{ + println(System.currentTimeMillis) + val cl = getClass.getClassLoader + println(System.currentTimeMillis) + cl + ClassLoader.getSystemClassLoader + } + */ + override def exportedClasspath = ClassPath(Seq(nailgunTarget, stage1Target) ) + override def exportedJars = ???//Seq[File]() + override def dependencies = ScalaDependencies(constants.scalaVersion).dependencies def targetClasspath = exportedClasspath } case class CbtDependency()(implicit val logger: Logger) extends Dependency{ - def exportedClasspath = ClassPath( Seq( stage2Target ) ) - def exportedJars = Seq[File]() + override def canBeCached = false + override def exportedClasspath = ClassPath( Seq( stage2Target ) ) + override def exportedJars = Seq[File]() override def dependencies = Seq( Stage1Dependency(), JavaDependency("net.incongru.watchservice","barbary-watchservice","1.0"), diff --git a/stage2/BuildBuild.scala b/stage2/BuildBuild.scala index 639154c..9746d8c 100644 --- a/stage2/BuildBuild.scala +++ b/stage2/BuildBuild.scala @@ -11,7 +11,11 @@ class BuildBuild(context: Context) extends Build(context){ exportedClasspath, classOf[BuildBuild].getClassLoader // FIXME: this looks wrong. Should be ClassLoader.getSystemClassLoader but that crashes ) - lib.create( lib.buildClassName )( managedContext )( cl ).asInstanceOf[Build] + cl + .loadClass(lib.buildClassName) + .getConstructor(classOf[Context]) + .newInstance(managedContext) + .asInstanceOf[Build] } override def triggerLoopFiles = super.triggerLoopFiles ++ managedBuild.triggerLoopFiles override def finalBuild = managedBuild.finalBuild diff --git a/stage2/mixins.scala b/stage2/mixins.scala index 753ee60..c3a57da 100644 --- a/stage2/mixins.scala +++ b/stage2/mixins.scala @@ -20,9 +20,6 @@ trait ScalaTest extends Build with Test{ "org.scalatest" %% "scalatest" % scalaTestVersion ) ++ super.dependencies - // workaround probable ScalaTest bug throwing away the outer classloader. Not caching doesn't nest them. - override def cacheDependencyClassLoader = false - override def run: ExitCode = { val discoveryPath = compile.toString++"/" context.logger.lib("discoveryPath: " ++ discoveryPath) |