package cbt
import java.nio.file._
import java.net._
import java.io._
import scala.collection.immutable.Seq
import scala.xml._
import paths._
import scala.concurrent._
import scala.concurrent.duration._
private final class Tree( val root: Dependency, computeChildren: => Seq[Tree] ){
lazy val children = computeChildren
def linearize: Seq[Dependency] = root +: children.flatMap(_.linearize)
def show(indent: Int = 0): Stream[Char] = {
(" " * indent ++ root.show ++ "\n").toStream #::: children.map(_.show(indent+1)).foldLeft(Stream.empty[Char])(_ #::: _)
}
}
trait ArtifactInfo extends Dependency{
def artifactId: String
def groupId: String
def version: String
protected def str = s"$groupId:$artifactId:$version"
override def show = super.show ++ s"($str)"
}
abstract class Dependency{
implicit def logger: Logger
protected def lib = new Stage1Lib(logger)
def needsUpdate: Boolean
//def cacheClassLoader: Boolean = false
private[cbt] def targetClasspath: ClassPath
def exportedClasspath: ClassPath
def exportedJars: Seq[File]
def jars: Seq[File] = exportedJars ++ dependencyJars
def canBeCached = false
//private type BuildCache = KeyLockedLazyCache[Dependency, Future[ClassPath]]
def exportClasspathConcurrently: ClassPath = {
// FIXME: this should separate a blocking and a non-blocking EC
import scala.concurrent.ExecutionContext.Implicits.global
Await.result(
exportClasspathConcurrently(
transitiveDependencies
.collect{ case d: ArtifactInfo => d }
.groupBy( d => (d.groupId,d.artifactId) )
.mapValues( _.head )
//, new BuildCache
),
Duration.Inf
)
}
def concurrencyEnabled = false
/**
The implementation of this method is untested and likely buggy
at this stage.
*/
private object cacheExportClasspathConcurrently extends Cache[Future[ClassPath]]
private def exportClasspathConcurrently(
latest: Map[(String, String),ArtifactInfo]//, cache: BuildCache
)( implicit ec: ExecutionContext ): Future[ClassPath] = cacheExportClasspathConcurrently{
Future.sequence( // trigger compilation / download of all dependencies first
this.dependencies.map{
d =>
// find out latest version of the required dependency
val l = d match {
case m: JavaDependency => latest( (m.groupId,m.artifactId) )
case _ => d
}
// // trigger compilation if not already triggered
// cache.get( l, l.exportClasspathConcurrently( latest, cache ) )
l.exportClasspathConcurrently( latest )
}
).map(
// merge dependency classpaths into one
ClassPath.flatten(_)
).map(
_ =>
// now that all dependencies are done, compile the code of this
exportedClasspath
)
}
private def actual(current: Dependency, latest: Map[(String,String),Dependency]) = current match {
case d: ArtifactInfo => latest((d.groupId,d.artifactId))
case d => d
}
private def dependencyClassLoader( latest: Map[(String,String),Dependency], cache: ClassLoaderCache ): ClassLoader = {
if( dependencies.isEmpty ){
ClassLoader.getSystemClassLoader
} else if( dependencies.size == 1 ){
dependencies.head.classLoaderRecursion( latest, cache )
} else if( dependencies.forall(_.canBeCached) ){
assert(transitiveDependencies.forall(_.canBeCached))
cache.persistent.get(
dependencyClasspath.string,
new MultiClassLoader(
dependencies.map( _.classLoaderRecursion(latest, cache) )
)
)
} else {
val (cachable, nonCachable) = dependencies.partition(_.canBeCached)
new URLClassLoader(
ClassPath.flatten( nonCachable.map(actual(_,latest)).map(_.exportedClasspath) ),
cache.persistent.get(
ClassPath.flatten( cachable.map(actual(_,latest)).map(_.exportedClasspath) ).string,
new MultiClassLoader(
cachable.map( _.classLoaderRecursion(latest, cache) )
)
)
)
new MultiClassLoader(
dependencies.map( _.classLoaderRecursion(latest, cache) )
)
}
}
protected def classLoaderRecursion( latest: Map[(String,String),Dependency], cache: ClassLoaderCache ): ClassLoader = {
if( canBeCached ){
val a = actual( this, latest )
cache.persistent.get(
a.classpath.string,
new cbt.URLClassLoader( a.exportedClasspath, dependencyClassLoader(latest, cache) )
)
} else {
new cbt.URLClassLoader( exportedClasspath, dependencyClassLoader(latest, cache) )
}
}
private object classLoaderCache extends Cache[ClassLoader]
def classLoader( cache: ClassLoaderCache ): ClassLoader = classLoaderCache{
if( concurrencyEnabled ){
// trigger concurrent building / downloading dependencies
exportClasspathConcurrently
}
classLoaderRecursion(
(this +: transitiveDependencies).collect{
case d: ArtifactInfo => d
}.groupBy(
d => (d.groupId,d.artifactId)
).mapValues(_.head),
cache
)
}
def classpath : ClassPath = exportedClasspath ++ dependencyClasspath
def dependencyJars : Seq[File] = transitiveDependencies.flatMap(_.jars)
def dependencyClasspath : ClassPath = ClassPath.flatten( transitiveDependencies.map(_.exportedClasspath) )
def dependencies: Seq[Dependency]
private def resolveRecursive(parents: List[Dependency] = List()): Tree = {
// diff removes circular dependencies
new Tree(this, (dependencies diff parents).map(_.resolveRecursive(this :: parents)))
}
def linearize(deps: Seq[Dependency]): Seq[Dependency] =
if(deps.isEmpty) deps else ( deps ++ linearize(deps.flatMap(_.dependencies)) )
private object transitiveDependenciesCache extends Cache[Seq[Dependency]]
/** return dependencies in order of linearized dependence. this is a bit tricky. */
def transitiveDependencies: Seq[Dependency] = transitiveDependenciesCache{
val deps = linearize(dependencies)
val hasInfo = deps.collect{ case d:ArtifactInfo => d }
val noInfo = deps.filter{
case _:ArtifactInfo => false
case _ => true
}
noInfo ++ JavaDependency.updateOutdated( hasInfo ).reverse.distinct
}
def show: String = this.getClass.getSimpleName
// ========== debug ==========
def dependencyTree: String = dependencyTreeRecursion()
private def dependencyTreeRecursion(indent: Int = 0): String = (
( " " * indent )
++ (if(needsUpdate) lib.red(show) else show)
++ dependencies.map(
"\n" ++ _.dependencyTreeRecursion(indent + 1)
).mkString
)
}
// TODO: all this hard codes the scala version, needs more flexibility
class ScalaCompilerDependency(version: String)(implicit logger: Logger) extends JavaDependency("org.scala-lang","scala-compiler",version)
class ScalaLibraryDependency (version: String)(implicit logger: Logger) extends JavaDependency("org.scala-lang","scala-library",version)
class ScalaReflectDependency (version: String)(implicit logger: Logger) extends JavaDependency("org.scala-lang","scala-reflect",version)
case class ScalaDependencies(version: String)(implicit val logger: Logger) extends Dependency{ sd =>
override final val needsUpdate = false
override def canBeCached = true
def targetClasspath = ClassPath(Seq())
def exportedClasspath = ClassPath(Seq())
def exportedJars = Seq[File]()
def dependencies = Seq(
new ScalaCompilerDependency(version),
new ScalaLibraryDependency(version),
new ScalaReflectDependency(version)
)
}
case class BinaryDependency( path: File, dependencies: Seq[Dependency] )(implicit val logger: Logger) extends Dependency{
def exportedClasspath = ClassPath(Seq(path))
def exportedJars = Seq[File](path)
override def needsUpdate = false
def targetClasspath = exportedClasspath
}
/** Allows to easily assemble a bunch of dependencies */
case class Dependencies( _dependencies: Dependency* )(implicit val logger: Logger) extends Dependency{
override def dependencies = _dependencies.to
def needsUpdate = dependencies.exists(_.needsUpdate)
def exportedClasspath = ClassPath(Seq())
def exportedJars = Seq()
def targetClasspath = ClassPath(Seq())
}
case class Stage1Dependency()(implicit val logger: Logger) extends Dependency{
def needsUpdate = false // FIXME: think this through, might allow simplifications and/or optimizations
override def canBeCached = false
/*
private object classLoaderRecursionCache extends Cache[ClassLoader]
override def classLoaderRecursion(latest: Map[(String,String),Dependency], cache: ClassLoaderCache) = classLoaderRecursionCache{
println(System.currentTimeMillis)
val cl = getClass.getClassLoader
println(System.currentTimeMillis)
cl
ClassLoader.getSystemClassLoader
}
*/
override def targetClasspath = exportedClasspath
override def exportedClasspath = ClassPath( Seq(nailgunTarget, stage1Target) )
override def exportedJars = ???//Seq[File]()
override def dependencies = Seq(
JavaDependency("org.scala-lang","scala-library",constants.scalaVersion),
JavaDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,"1.0.5")
)
// FIXME: implement sanity check to prevent using incompatible scala-library and xml version on cp
override def classLoaderRecursion( latest: Map[(String,String),Dependency], cache: ClassLoaderCache )
= getClass.getClassLoader
}
case class CbtDependency()(implicit val logger: Logger) extends Dependency{
def needsUpdate = false // FIXME: think this through, might allow simplifications and/or optimizations
override def canBeCached = false
override def targetClasspath = exportedClasspath
override def exportedClasspath = ClassPath( Seq( stage2Target ) )
override def exportedJars = ???
override def dependencies = Seq(
Stage1Dependency(),
JavaDependency("net.incongru.watchservice","barbary-watchservice","1.0"),
JavaDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r")
)
}
case class Classifier(name: Option[String])
object Classifier{
object none extends Classifier(None)
object javadoc extends Classifier(Some("javadoc"))
object sources extends Classifier(Some("sources"))
}
case class JavaDependency(
groupId: String, artifactId: String, version: String, classifier: Classifier = Classifier.none
)(implicit val logger: Logger) extends ArtifactInfo{
assert(
Option(groupId).collect{
case JavaDependency.ValidIdentifier(_) =>
}.nonEmpty,
s"not a valid groupId: '$groupId'"
)
assert(
Option(artifactId).collect{
case JavaDependency.ValidIdentifier(_) =>
}.nonEmpty,
s"not a valid artifactId: '$artifactId'"
)
assert(
version != "" && version != null && !version.startsWith(" ") && !version.endsWith(" "),
s"not a valid version: '$version'"
)
override def needsUpdate = false
override def canBeCached = true
private val groupPath = groupId.split("\\.").mkString("/")
def basePath = s"/$groupPath/$artifactId/$version/$artifactId-$version" ++ classifier.name.map("-"++_).getOrElse("")
private def resolverUrl:URL = new URL(
if(version.endsWith("-SNAPSHOT")) "https://oss.sonatype.org/content/repositories/snapshots" else "https://repo1.maven.org/maven2"
)
private def baseUrl: URL = resolverUrl ++ basePath
private def baseFile: File = mavenCache ++ basePath
private def pomFile: File = baseFile ++ ".pom"
private def jarFile: File = baseFile ++ ".jar"
//private def coursierJarFile = userHome++"/.coursier/cache/v1/https/repo1.maven.org/maven2"++basePath++".jar"
private def pomUrl: URL = baseUrl ++ ".pom"
private[cbt] def jarUrl: URL = baseUrl ++ ".jar"
def exportedJars = Seq( jar )
def exportedClasspath = ClassPath( exportedJars )
def targetClasspath = exportedClasspath
import scala.collection.JavaConversions._
def jarSha1 = {
val file = jarFile ++ ".sha1"
lib.download( jarUrl ++ ".sha1" , file, None )
// split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c jna-3.2.2.pom
Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim
}
def pomSha1 = {
val file = pomFile++".sha1"
lib.download( pomUrl++".sha1" , file, None )
// split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c jna-3.2.2.pom
Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim
}
private object jarCache extends Cache[File]
def jar = jarCache{
lib.download( jarUrl, jarFile, Some(jarSha1) )
jarFile
}
def pomXml = XML.loadFile(pom.toString)
def pom = {
lib.download( pomUrl, pomFile, Some(pomSha1) )
pomFile
}
// ========== pom traversal ==========
lazy val transitivePom: Seq[JavaDependency] = {
(pomXml \ "parent").collect{
case parent =>
JavaDependency(
(parent \ "groupId").text,
(parent \ "artifactId").text,
(parent \ "version").text
)(logger)
}.flatMap(_.transitivePom) :+ this
}
lazy val properties: Map[String, String] = (
transitivePom.flatMap{ d =>
val props = (d.pomXml \ "properties").flatMap(_.child).map{
tag => tag.label -> tag.text
}
logger.pom(s"Found properties in $pom: $props")
props
}
).toMap
lazy val dependencyVersions: Map[String, (String,String)] =
transitivePom.flatMap(
p =>
(p.pomXml \ "dependencyManagement" \ "dependencies" \ "dependency").map{
xml =>
val groupId = p.lookup(xml,_ \ "groupId").get
val artifactId = p.lookup(xml,_ \ "artifactId").get
val version = p.lookup(xml,_ \ "version").get
artifactId -> (groupId, version)
}
).toMap
def dependencies: Seq[JavaDependency] = {
if(classifier == Classifier.sources) Seq()
else (pomXml \ "dependencies" \ "dependency").collect{
case xml if (xml \ "scope").text == "" && (xml \ "optional").text != "true" =>
val artifactId = lookup(xml,_ \ "artifactId").get
val groupId =
lookup(xml,_ \ "groupId").getOrElse(
dependencyVersions
.get(artifactId).map(_._1)
.getOrElse(
throw new Exception(s"$artifactId not found in \n$dependencyVersions")
)
)
val version =
lookup(xml,_ \ "version").getOrElse(
dependencyVersions
.get(artifactId).map(_._2)
.getOrElse(
throw new Exception(s"$artifactId not found in \n$dependencyVersions")
)
)
JavaDependency(
groupId, artifactId, version,
Classifier( Some( (xml \ "classifier").text ).filterNot(_ == "").filterNot(_ == null) )
)
}.toVector
}
def lookup( xml: Node, accessor: Node => NodeSeq ): Option[String] = {
//println("lookup in "++pomUrl)
val Substitution = "\\$\\{([^\\}]+)\\}".r
accessor(xml).headOption.flatMap{v =>
//println("found: "++v.text)
v.text match {
case Substitution(path) => Option(
properties.get(path).orElse(
transitivePom.reverse.flatMap{ d =>
Some(path.split("\\.").toList).collect{
case "project" :: path =>
path.foldLeft(d.pomXml:NodeSeq){ case (xml,tag) => xml \ tag }.text
}.filter(_ != "")
}.headOption
)
.getOrElse( throw new Exception(s"Can't find $path in \n$properties.\n\npomParents: $transitivePom\n\n pomXml:\n$pomXml" )))
//println("lookup "++path ++ ": "++(pomXml\path).text)
case value => Option(value)
}
}
}
}
object JavaDependency{
def ValidIdentifier = "^([A-Za-z0-9_\\-.]+)$".r // according to maven's DefaultModelValidator.java
def semanticVersionLessThan(left: String, right: String) = {
// FIXME: this ignores ends when different size
val zipped = left.split("\\.|\\-").map(toInt) zip right.split("\\.|\\-").map(toInt)
val res = zipped.map {
case (Left(i),Left(j)) => i compare j
case (Right(i),Right(j)) => i compare j
case (Left(i),Right(j)) => i.toString compare j
case (Right(i),Left(j)) => i compare j.toString
}
res.find(_ != 0).map(_ < 0).getOrElse(false)
}
def toInt(str: String): Either[Int,String] = try {
Left(str.toInt)
} catch {
case e: NumberFormatException => Right(str)
}
/* this obviously should be overridable somehow */
def updateOutdated(
deps: Seq[ArtifactInfo],
versionLessThan: (String, String) => Boolean = semanticVersionLessThan
)(implicit logger: Logger): Seq[ArtifactInfo] = {
val latest = deps
.groupBy( d => (d.groupId, d.artifactId) )
.mapValues(
_.sortBy( _.version )( Ordering.fromLessThan(versionLessThan) )
.last
)
deps.map{
d =>
val l = latest((d.groupId,d.artifactId))
if(d != l) logger.resolver("outdated: "++d.show)
l
}
}
}