package cbt import java.nio.file._ import java.nio.charset.StandardCharsets import java.net._ import java.io._ import scala.xml._ import scala.concurrent._ import scala.concurrent.duration._ trait DependencyImplementation extends Dependency{ implicit protected def logger: Logger protected def lib = new Stage1Lib(logger) /** CAREFUL: this is never allowed to return true for the same dependency more than once in a single cbt run. Otherwise we can end up with multiple classloaders for the same classes since classLoaderRecursion recreates the classLoader when it sees this flag and it can be called multiple times. Maybe we can find a safer solution than this current state. */ def needsUpdate: Boolean //def cacheClassLoader: Boolean = false private[cbt] def targetClasspath: ClassPath def dependencyClasspathArray: Array[File] = dependencyClasspath.files.toArray def exportedClasspathArray: Array[File] = exportedClasspath.files.toArray def exportedClasspath: ClassPath def dependenciesArray: Array[Dependency] = dependencies.to def needsUpdateCompat: java.lang.Boolean = needsUpdate /* //private type BuildCache = KeyLockedLazyCache[Dependency, Future[ClassPath]] def exportClasspathConcurrently: ClassPath = { // FIXME: this should separate a blocking and a non-blocking EC import scala.concurrent.ExecutionContext.Implicits.global Await.result( exportClasspathConcurrently( transitiveDependencies .collect{ case d: ArtifactInfo => d } .groupBy( d => (d.groupId,d.artifactId) ) .mapValues( _.head ) //, new BuildCache ), // FIXME Duration.Inf ) } def concurrencyEnabled = false /** The implementation of this method is untested and likely buggy at this stage. */ def exportClasspathConcurrently( latest: Map[(String, String),Dependency with ArtifactInfo]//, cache: BuildCache )( implicit ec: ExecutionContext ): Future[AnyRef] = { Future.sequence( // trigger compilation / download of all dependencies first this.dependencies.map{ d => // find out latest version of the required dependency val l = d match { case m: BoundMavenDependency => latest( (m.groupId,m.artifactId) ) case _ => d } // // trigger compilation if not already triggered // cache.get( l, l.exportClasspathConcurrently( latest, cache ) ) l.exportClasspathConcurrently( latest ) // FIXME } ).map( // merge dependency classpaths into one ClassPath.flatten(_) ).map( _ => // now that all dependencies are done, compile the code of this exportedClasspath ) } */ def classLoader( cache: ClassLoaderCache ): ClassLoader = { /* if( concurrencyEnabled ){ // trigger concurrent building / downloading dependencies exportClasspathConcurrently } */ lib.classLoaderRecursion( this, (this +: transitiveDependencies).collect{ case d: ArtifactInfo => d }.groupBy( d => (d.groupId,d.artifactId) ).mapValues(_.head), cache // FIXME ) } // FIXME: these probably need to update outdated as well def classpath : ClassPath = exportedClasspath ++ dependencyClasspath def dependencyClasspath : ClassPath = ClassPath( transitiveDependencies .flatMap(_.exportedClasspath.files) .distinct // <- currently needed here to handle diamond dependencies on builds (duplicate in classpath) ) def dependencies: Seq[Dependency] private object transitiveDependenciesCache extends Cache[Seq[Dependency]] /** return dependencies in order of linearized dependence. this is a bit tricky. */ def transitiveDependencies: Seq[Dependency] = transitiveDependenciesCache{ lib.transitiveDependencies(this) } override def show: String = this.getClass.getSimpleName // ========== debug ========== def dependencyTree: String = lib.dependencyTreeRecursion(this) } // TODO: all this hard codes the scala version, needs more flexibility class ScalaCompilerDependency(cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-compiler",version, Classifier.none), Seq(mavenCentral)) class ScalaLibraryDependency (cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-library",version, Classifier.none), Seq(mavenCentral)) class ScalaReflectDependency (cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-reflect",version, Classifier.none), Seq(mavenCentral)) case class ScalaDependencies(cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit val logger: Logger) extends DependencyImplementation{ sd => override final val needsUpdate = false def targetClasspath = ClassPath() def exportedClasspath = ClassPath() def dependencies = Seq( new ScalaCompilerDependency(cbtHasChanged, mavenCache, version), new ScalaLibraryDependency(cbtHasChanged, mavenCache, version), new ScalaReflectDependency(cbtHasChanged, mavenCache, version) ) } case class BinaryDependency( path: File, dependencies: Seq[Dependency] )(implicit val logger: Logger) extends DependencyImplementation{ def exportedClasspath = ClassPath(Seq(path)) override def needsUpdate = false def targetClasspath = exportedClasspath } /** Allows to easily assemble a bunch of dependencies */ case class Dependencies( dependencies: Seq[Dependency] )(implicit val logger: Logger) extends DependencyImplementation{ override def needsUpdate = dependencies.exists(_.needsUpdate) override def exportedClasspath = ClassPath() override def targetClasspath = ClassPath() } case class Stage1Dependency(cbtHasChanged: Boolean, mavenCache: File, nailgunTarget: File, stage1Target: File, compatibilityTarget: File)(implicit val logger: Logger) extends DependencyImplementation{ override def needsUpdate = false override def targetClasspath = exportedClasspath override def exportedClasspath = ClassPath( Seq(nailgunTarget, stage1Target) ) val compatibilityDependency = CompatibilityDependency(cbtHasChanged, compatibilityTarget) override def dependencies = Seq( compatibilityDependency ) ++ MavenResolver(cbtHasChanged,mavenCache,mavenCentral).bind( MavenDependency("org.scala-lang","scala-library",constants.scalaVersion), MavenDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,constants.scalaXmlVersion) ) } case class CompatibilityDependency(cbtHasChanged: Boolean, compatibilityTarget: File)(implicit val logger: Logger) extends DependencyImplementation{ override def needsUpdate = false override def targetClasspath = exportedClasspath override def exportedClasspath = ClassPath( Seq(compatibilityTarget) ) override def dependencies = Seq() } case class CbtDependency(cbtHasChanged: Boolean, mavenCache: File, nailgunTarget: File, stage1Target: File, stage2Target: File, compatibilityTarget: File)(implicit val logger: Logger) extends DependencyImplementation{ override def needsUpdate = false override def targetClasspath = exportedClasspath override def exportedClasspath = ClassPath( Seq( stage2Target ) ) val stage1Dependency = Stage1Dependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, compatibilityTarget) override def dependencies = Seq( stage1Dependency ) ++ MavenResolver(cbtHasChanged, mavenCache,mavenCentral).bind( MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0"), MavenDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r") ) } case class Classifier(name: Option[String]) object Classifier{ object none extends Classifier(None) object javadoc extends Classifier(Some("javadoc")) object sources extends Classifier(Some("sources")) } abstract class DependenciesProxy{ } class BoundMavenDependencies( cbtHasChanged: Boolean, mavenCache: File, urls: Seq[URL], mavenDependencies: Seq[MavenDependency] )(implicit logger: Logger) extends Dependencies( mavenDependencies.map( BoundMavenDependency(cbtHasChanged,mavenCache,_,urls) ) ) case class MavenDependency( groupId: String, artifactId: String, version: String, classifier: Classifier = Classifier.none ){ private[cbt] def serialize = groupId ++ ":" ++ artifactId ++ ":"++ version ++ classifier.name.map(":" ++ _).getOrElse("") } object MavenDependency{ private[cbt] def deserialize = (_:String).split(":") match { case col => MavenDependency( col(0), col(1), col(2), Classifier(col.lift(3)) ) } } // FIXME: take MavenResolver instead of mavenCache and repositories separately case class BoundMavenDependency( cbtHasChanged: Boolean, mavenCache: File, mavenDependency: MavenDependency, repositories: Seq[URL] )(implicit val logger: Logger) extends ArtifactInfo with DependencyImplementation{ val MavenDependency( groupId, artifactId, version, classifier ) = mavenDependency assert( Option(groupId).collect{ case BoundMavenDependency.ValidIdentifier(_) => }.nonEmpty, s"not a valid groupId: '$groupId'" ) assert( Option(artifactId).collect{ case BoundMavenDependency.ValidIdentifier(_) => }.nonEmpty, s"not a valid artifactId: '$artifactId'" ) assert( version != "" && version != null && !version.startsWith(" ") && !version.endsWith(" "), s"not a valid version: '$version'" ) override def show: String = this.getClass.getSimpleName ++ "(" ++ mavenDependency.serialize ++ ")" override def needsUpdate = false private val groupPath = groupId.split("\\.").mkString("/") protected[cbt] def basePath = s"/$groupPath/$artifactId/$version/$artifactId-$version" ++ classifier.name.map("-"++_).getOrElse("") //private def coursierJarFile = userHome++"/.coursier/cache/v1/https/repo1.maven.org/maven2"++basePath++".jar" def exportedJars = Seq( jar ) override def exportedClasspath = ClassPath( exportedJars ) override def targetClasspath = exportedClasspath import scala.collection.JavaConversions._ private def resolve(suffix: String, hash: Option[String]): File = { logger.resolver("Resolving "+this) val file = mavenCache ++ basePath ++ "." ++ suffix val urls = repositories.map(_ ++ basePath ++ "." ++ suffix) urls.find( lib.download(_, file, hash) ).getOrElse( throw new Exception(s"\nCannot resolve\n$this\nCan't find any of\n"++urls.mkString("\n")) ) file } private def resolveHash(suffix: String) = { Files.readAllLines( resolve( suffix ++ ".sha1", None ).toPath, StandardCharsets.UTF_8 ).mkString("\n").split(" ").head.trim } private object jarSha1Cache extends Cache[String] def jarSha1: String = jarSha1Cache{ resolveHash("jar") } private object pomSha1Cache extends Cache[String] def pomSha1: String = pomSha1Cache{ resolveHash("pom") } private object jarCache extends Cache[File] def jar: File = jarCache{ resolve("jar", Some(jarSha1)) } private object pomCache extends Cache[File] def pom: File = pomCache{ resolve("pom", Some(pomSha1)) } private def pomXml = XML.loadFile(pom.string) // ========== pom traversal ========== private lazy val transitivePom: Seq[BoundMavenDependency] = { (pomXml \ "parent").collect{ case parent => BoundMavenDependency( cbtHasChanged: Boolean, mavenCache, MavenDependency( (parent \ "groupId").text, (parent \ "artifactId").text, (parent \ "version").text ), repositories )(logger) }.flatMap(_.transitivePom) :+ this } private lazy val properties: Map[String, String] = ( transitivePom.flatMap{ d => val props = (d.pomXml \ "properties").flatMap(_.child).map{ tag => tag.label -> tag.text } logger.pom(s"Found properties in $pom: $props") props } ).toMap private lazy val dependencyVersions: Map[String, (String,String)] = transitivePom.flatMap( p => (p.pomXml \ "dependencyManagement" \ "dependencies" \ "dependency").map{ xml => val groupId = p.lookup(xml,_ \ "groupId").get val artifactId = p.lookup(xml,_ \ "artifactId").get val version = p.lookup(xml,_ \ "version").get artifactId -> (groupId, version) } ).toMap def dependencies: Seq[BoundMavenDependency] = { if(classifier == Classifier.sources) Seq() else { lib.cacheOnDisk( cbtHasChanged, mavenCache ++ basePath ++ ".pom.dependencies" )( MavenDependency.deserialize )( _.serialize ){ (pomXml \ "dependencies" \ "dependency").collect{ case xml if ( (xml \ "scope").text == "" || (xml \ "scope").text == "compile" ) && (xml \ "optional").text != "true" => val artifactId = lookup(xml,_ \ "artifactId").get val groupId = lookup(xml,_ \ "groupId").getOrElse( dependencyVersions .get(artifactId).map(_._1) .getOrElse( throw new Exception(s"$artifactId not found in \n$dependencyVersions") ) ) val version = lookup(xml,_ \ "version").getOrElse( dependencyVersions .get(artifactId).map(_._2) .getOrElse( throw new Exception(s"$artifactId not found in \n$dependencyVersions") ) ) val classifier = Classifier( Some( (xml \ "classifier").text ).filterNot(_ == "").filterNot(_ == null) ) MavenDependency( groupId, artifactId, version, classifier ) }.toVector }.map( BoundMavenDependency( cbtHasChanged, mavenCache, _, repositories ) ).to } } def lookup( xml: Node, accessor: Node => NodeSeq ): Option[String] = { //println("lookup in "++pomUrl) val Substitution = "\\$\\{([^\\}]+)\\}".r accessor(xml).headOption.map{v => //println("found: "++v.text) Substitution.replaceAllIn( v.text, matcher => { val path = matcher.group(1) properties.get(path).orElse( transitivePom.reverse.flatMap{ d => Some(path.split("\\.").toList).collect{ case "project" :: path => path.foldLeft(d.pomXml:NodeSeq){ case (xml,tag) => xml \ tag }.text }.filter(_ != "") }.headOption ) .getOrElse( throw new Exception(s"Can't find $path in \n$properties.\n\npomParents: $transitivePom\n\n pomXml:\n$pomXml" ) ) //println("lookup "++path ++ ": "++(pomXml\path).text) } ) } } } object BoundMavenDependency{ def ValidIdentifier = "^([A-Za-z0-9_\\-.]+)$".r // according to maven's DefaultModelValidator.java def semanticVersionLessThan(left: Array[Either[Int,String]], right: Array[Either[Int,String]]) = { // FIXME: this ignores ends when different size val zipped = left zip right val res = zipped.map { case (Left(i),Left(j)) => i compare j case (Right(i),Right(j)) => i compare j case (Left(i),Right(j)) => i.toString compare j case (Right(i),Left(j)) => i compare j.toString } res.find(_ != 0).map(_ < 0).getOrElse(false) } def toInt(str: String): Either[Int,String] = try { Left(str.toInt) } catch { case e: NumberFormatException => Right(str) } /* this obviously should be overridable somehow */ def updateOutdated( deps: Seq[Dependency with ArtifactInfo], versionLessThan: (Array[Either[Int,String]], Array[Either[Int,String]]) => Boolean = semanticVersionLessThan )(implicit logger: Logger): Seq[Dependency with ArtifactInfo] = { val latest = deps .groupBy( d => (d.groupId, d.artifactId) ) .mapValues( _.groupBy(_.version) // remove duplicates .map( _._2.head ) .toVector .sortBy( _.version.split("\\.|\\-").map(toInt) )( Ordering.fromLessThan(versionLessThan) ) .last ) deps.map{ d => val l = latest((d.groupId,d.artifactId)) if(d != l) logger.resolver("outdated: "++d.show) l } } }