diff options
Diffstat (limited to 'stage1')
-rw-r--r-- | stage1/KeyLockedLazyCache.scala | 33 | ||||
-rw-r--r-- | stage1/Stage1Lib.scala | 27 | ||||
-rw-r--r-- | stage1/constants.scala | 3 | ||||
-rw-r--r-- | stage1/resolver.scala | 111 |
4 files changed, 140 insertions, 34 deletions
diff --git a/stage1/KeyLockedLazyCache.scala b/stage1/KeyLockedLazyCache.scala new file mode 100644 index 0000000..c8b37ea --- /dev/null +++ b/stage1/KeyLockedLazyCache.scala @@ -0,0 +1,33 @@ +/* +package cbt +import java.util.concurrent.ConcurrentHashMap +import scala.concurrent.Future + +/** +A cache that lazily computes values if needed during lookup. +Locking occurs on the key, so separate keys can be looked up +simultaneously without a deadlock. +*/ +final private[cbt] class KeyLockedLazyCache[Key <: AnyRef,Value]{ + private val keys = new ConcurrentHashMap[Key,LockableKey]() + private val builds = new ConcurrentHashMap[LockableKey,Value]() + + private class LockableKey + def get( key: Key, value: => Value ): Value = { + val keyObject = keys.synchronized{ + if( ! (keys containsKey key) ){ + keys.put( key, new LockableKey ) + } + keys get key + } + // synchronizing on key only, so asking for a particular key does + // not block the whole cache, but just that cache entry + key.synchronized{ + if( ! (builds containsKey keyObject) ){ + builds.put( keyObject, value ) + } + builds get keyObject + } + } +} +*/ diff --git a/stage1/Stage1Lib.scala b/stage1/Stage1Lib.scala index 9a1adb5..c9cd5c9 100644 --- a/stage1/Stage1Lib.scala +++ b/stage1/Stage1Lib.scala @@ -38,6 +38,9 @@ class BaseLib{ class Stage1Lib( val logger: Logger ) extends BaseLib{ lib => + implicit val implicitLogger: Logger = logger + + def scalaMajorVersion(scalaMinorVersion: String) = scalaMinorVersion.split("\\.").take(2).mkString(".") // ========== reflection ========== @@ -124,11 +127,11 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ private def getZincDependencyJar(zincDeps: Seq[Dependency], zincVersion: String, group: String, name: String) = { zincDeps .collect { - case dependency @ MavenDependency( group, name, _, false ) => + case dependency @ JavaDependency( group, name, _, _ ) => dependency } .headOption - .getOrElse( throw new Exception(s"cannot find $name in zinc $zincVersion dependencies") ) + .getOrElse( throw new Exception(s"cannot find $name in zinc $zincVersion dependencies: " ++ zincDeps.toString) ) .jar } @@ -150,15 +153,15 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ // only run zinc if files changed, for performance reasons // FIXME: this is broken, need invalidate on changes in dependencies as well if( needsRecompile ){ - val zinc = MavenDependency("com.typesafe.zinc","zinc", zincVersion)(logger) + val zinc = JavaDependency("com.typesafe.zinc","zinc", zincVersion) val zincDeps = zinc.transitiveDependencies - + val sbtInterface = getZincDependencyJar(zincDeps, zincVersion, "com.typesafe.sbt", "sbt-interface") val compilerInterface = getZincDependencyJar(zincDeps, zincVersion, "com.typesafe.sbt", "compiler-interface") - val scalaLibrary = MavenDependency("org.scala-lang","scala-library",scalaVersion)(logger).jar - val scalaReflect = MavenDependency("org.scala-lang","scala-reflect",scalaVersion)(logger).jar - val scalaCompiler = MavenDependency("org.scala-lang","scala-compiler",scalaVersion)(logger).jar + val scalaLibrary = JavaDependency("org.scala-lang","scala-library",scalaVersion).jar + val scalaReflect = JavaDependency("org.scala-lang","scala-reflect",scalaVersion).jar + val scalaCompiler = JavaDependency("org.scala-lang","scala-compiler",scalaVersion).jar val code = redirectOutToErr{ lib.runMain( @@ -228,4 +231,12 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ System.setSecurityManager(NailgunLauncher.defaultSecurityManager) } } -} + + def ScalaDependency( + groupId: String, artifactId: String, version: String, classifier: Classifier = Classifier.none, + scalaVersion: String + ) = + JavaDependency( + groupId, artifactId ++ "_" ++ scalaVersion, version, classifier + ) +}
\ No newline at end of file diff --git a/stage1/constants.scala b/stage1/constants.scala index bf0943e..a14754e 100644 --- a/stage1/constants.scala +++ b/stage1/constants.scala @@ -1,4 +1,5 @@ package cbt object constants{ - def scalaVersion = Option(System.getenv("SCALA_VERSION")).get + val scalaVersion = Option(System.getenv("SCALA_VERSION")).get + val scalaMajorVersion = scalaVersion.split("\\.").take(2).mkString(".") } diff --git a/stage1/resolver.scala b/stage1/resolver.scala index e86f1b3..bba8a70 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -5,6 +5,8 @@ import java.io._ import scala.collection.immutable.Seq import scala.xml._ import paths._ +import scala.concurrent._ +import scala.concurrent.duration._ private final class Tree( val root: Dependency, computeChildren: => Seq[Tree] ){ lazy val children = computeChildren @@ -35,8 +37,60 @@ abstract class Dependency{ def canBeCached = false def cacheDependencyClassLoader = true + //private type BuildCache = KeyLockedLazyCache[Dependency, Future[ClassPath]] + def exportClasspathConcurrently: ClassPath = { + // FIXME: this should separate a blocking and a non-blocking EC + import scala.concurrent.ExecutionContext.Implicits.global + Await.result( + exportClasspathConcurrently( + transitiveDependencies + .collect{ case d: ArtifactInfo => d } + .groupBy( d => (d.groupId,d.artifactId) ) + .mapValues( _.head ) + //, new BuildCache + ), + Duration.Inf + ) + } + + def concurrencyEnabled = false + + /** + The implementation of this method is untested and likely buggy + at this stage. + */ + private object cacheExportClasspathConcurrently extends Cache[Future[ClassPath]] + private def exportClasspathConcurrently( + latest: Map[(String, String),ArtifactInfo]//, cache: BuildCache + )( implicit ec: ExecutionContext ): Future[ClassPath] = cacheExportClasspathConcurrently{ + Future.sequence( // trigger compilation / download of all dependencies first + this.dependencies.map{ + d => + // find out latest version of the required dependency + val l = d match { + case m: JavaDependency => latest( (m.groupId,m.artifactId) ) + case _ => d + } + // // trigger compilation if not already triggered + // cache.get( l, l.exportClasspathConcurrently( latest, cache ) ) + l.exportClasspathConcurrently( latest ) + } + ).map( + // merge dependency classpaths into one + ClassPath.flatten(_) + ).map( + _ => + // now that all dependencies are done, compile the code of this + exportedClasspath + ) + } + private object classLoaderCache extends Cache[URLClassLoader] def classLoader: URLClassLoader = classLoaderCache{ + if( concurrencyEnabled ){ + // trigger concurrent building / downloading dependencies + exportClasspathConcurrently + } val transitiveClassPath = transitiveDependencies.map{ case d if d.canBeCached => Left(d) case d => Right(d) @@ -80,7 +134,7 @@ abstract class Dependency{ case _:ArtifactInfo => false case _ => true } - noInfo ++ MavenDependency.removeOutdated( hasInfo ) + noInfo ++ JavaDependency.removeOutdated( hasInfo ) } def show: String = this.getClass.getSimpleName @@ -96,9 +150,9 @@ abstract class Dependency{ } // TODO: all this hard codes the scala version, needs more flexibility -class ScalaCompilerDependency(version: String)(implicit logger: Logger) extends MavenDependency("org.scala-lang","scala-compiler",version) -class ScalaLibraryDependency (version: String)(implicit logger: Logger) extends MavenDependency("org.scala-lang","scala-library",version) -class ScalaReflectDependency (version: String)(implicit logger: Logger) extends MavenDependency("org.scala-lang","scala-reflect",version) +class ScalaCompilerDependency(version: String)(implicit logger: Logger) extends JavaDependency("org.scala-lang","scala-compiler",version) +class ScalaLibraryDependency (version: String)(implicit logger: Logger) extends JavaDependency("org.scala-lang","scala-library",version) +class ScalaReflectDependency (version: String)(implicit logger: Logger) extends JavaDependency("org.scala-lang","scala-reflect",version) case class ScalaDependencies(version: String)(implicit val logger: Logger) extends Dependency{ sd => final val updated = false @@ -106,9 +160,9 @@ case class ScalaDependencies(version: String)(implicit val logger: Logger) exten def exportedClasspath = ClassPath(Seq()) def exportedJars = Seq[File]() def dependencies = Seq( - new ScalaCompilerDependency(version)(logger), - new ScalaLibraryDependency(version)(logger), - new ScalaReflectDependency(version)(logger) + new ScalaCompilerDependency(version), + new ScalaLibraryDependency(version), + new ScalaReflectDependency(version) ) } @@ -128,27 +182,34 @@ case class CbtDependency()(implicit val logger: Logger) extends Dependency{ def exportedClasspath = ClassPath( Seq( stage2Target ) ) def exportedJars = Seq[File]() override def dependencies = Seq( - Stage1Dependency()(logger), - MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0")(logger), - MavenDependency("com.lihaoyi","ammonite-repl_2.11.7","0.5.5")(logger), - MavenDependency("org.scala-lang.modules","scala-xml_2.11","1.0.5")(logger) + Stage1Dependency(), + JavaDependency("net.incongru.watchservice","barbary-watchservice","1.0"), + lib.ScalaDependency( + "com.lihaoyi","ammonite-ops","0.5.5", scalaVersion = constants.scalaMajorVersion + ), + lib.ScalaDependency( + "org.scala-lang.modules","scala-xml","1.0.5", scalaVersion = constants.scalaMajorVersion + ) ) def updated = false // FIXME: think this through, might allow simplifications and/or optimizations } -sealed trait ClassifierBase -final case class Classifier(name: String) extends ClassifierBase -case object javadoc extends ClassifierBase -case object sources extends ClassifierBase +case class Classifier(name: Option[String]) +object Classifier{ + object none extends Classifier(None) + object javadoc extends Classifier(Some("javadoc")) + object sources extends Classifier(Some("sources")) +} -case class MavenDependency( groupId: String, artifactId: String, version: String, sources: Boolean = false )(implicit val logger: Logger) - extends ArtifactInfo{ +case class JavaDependency( + groupId: String, artifactId: String, version: String, classifier: Classifier = Classifier.none +)(implicit val logger: Logger) extends ArtifactInfo{ def updated = false override def canBeCached = true private val groupPath = groupId.split("\\.").mkString("/") - def basePath = s"/$groupPath/$artifactId/$version/$artifactId-$version"++(if(sources) "-sources" else "") + def basePath = s"/$groupPath/$artifactId/$version/$artifactId-$version" ++ classifier.name.map("-"++_).getOrElse("") private def resolverUrl:URL = new URL( if(version.endsWith("-SNAPSHOT")) "https://oss.sonatype.org/content/repositories/snapshots" else "https://repo1.maven.org/maven2" @@ -197,25 +258,25 @@ case class MavenDependency( groupId: String, artifactId: String, version: String // ========== pom traversal ========== - lazy val pomParents: Seq[MavenDependency] = { + lazy val pomParents: Seq[JavaDependency] = { (pomXml \ "parent").collect{ case parent => - MavenDependency( + JavaDependency( (parent \ "groupId").text, (parent \ "artifactId").text, (parent \ "version").text )(logger) } } - def dependencies: Seq[MavenDependency] = { - if(sources) Seq() + def dependencies: Seq[JavaDependency] = { + if(classifier == Classifier.sources) Seq() else (pomXml \ "dependencies" \ "dependency").collect{ case xml if (xml \ "scope").text == "" && (xml \ "optional").text != "true" => - MavenDependency( + JavaDependency( lookup(xml,_ \ "groupId").get, lookup(xml,_ \ "artifactId").get, lookup(xml,_ \ "version").get, - (xml \ "classifier").text == "sources" + Classifier( Some( (xml \ "classifier").text ).filterNot(_ == "").filterNot(_ == null) ) )(logger) }.toVector } @@ -235,7 +296,7 @@ case class MavenDependency( groupId: String, artifactId: String, version: String ) } } -object MavenDependency{ +object JavaDependency{ def semanticVersionLessThan(left: String, right: String) = { // FIXME: this ignores ends when different size val zipped = left.split("\\.|\\-").map(toInt) zip right.split("\\.|\\-").map(toInt) |