package cbt import java.nio.file._ import java.nio.charset.StandardCharsets import java.net._ import java.io._ import scala.xml._ trait DependencyImplementation extends Dependency{ implicit protected def logger: Logger protected def lib = new Stage1Lib(logger) implicit protected def transientCache: java.util.Map[AnyRef,AnyRef] implicit protected def classLoaderCache: ClassLoaderCache /** key used by taskCache to identify different objects that represent the same logical module */ protected def moduleKey: String /** caches given value in context keyed with given key and projectDirectory the context is fresh on every complete run of cbt */ protected lazy val taskCache = new PerClassCache(transientCache, moduleKey) private[cbt] def targetClasspath: ClassPath def exportedClasspathArray: Array[File] = exportedClasspath.files.toArray def exportedClasspath: ClassPath def dependenciesArray: Array[Dependency] = dependencies.to @deprecated("this method was replaced by lastModifiedCompat","") def needsUpdateCompat = true @deprecated("this method was replaced by dependenciesArray","") def dependencyClasspathArray = dependencyClasspath.files.toArray /* //private type BuildCache = KeyLockedLazyCache[Dependency, Future[ClassPath]] def exportClasspathConcurrently: ClassPath = { // FIXME: this should separate a blocking and a non-blocking EC import scala.concurrent.ExecutionContext.Implicits.global Await.result( exportClasspathConcurrently( transitiveDependencies .collect{ case d: ArtifactInfo => d } .groupBy( d => (d.groupId,d.artifactId) ) .mapValues( _.head ) //, new BuildCache ), // FIXME Duration.Inf ) } def concurrencyEnabled = false /** The implementation of this method is untested and likely buggy at this stage. */ def exportClasspathConcurrently( latest: Map[(String, String),Dependency with ArtifactInfo]//, cache: BuildCache )( implicit ec: ExecutionContext ): Future[AnyRef] = { Future.sequence( // trigger compilation / download of all dependencies first this.dependencies.map{ d => // find out latest version of the required dependency val l = d match { case m: BoundMavenDependency => latest( (m.groupId,m.artifactId) ) case _ => d } // // trigger compilation if not already triggered // cache.get( l, l.exportClasspathConcurrently( latest, cache ) ) l.exportClasspathConcurrently( latest ) // FIXME } ).map( // merge dependency classpaths into one ClassPath.flatten(_) ).map( _ => // now that all dependencies are done, compile the code of this exportedClasspath ) } */ def runMain( className: String, args: Seq[String] ): ExitCode = lib.getMain( classLoader.loadClass( className ) )( args ) def runMain( args: Seq[String] ): ExitCode = runMain( mainClassOrFail.getName, args ) def mainClassOrFail = mainClass.getOrElse( throw new RuntimeException( "No main class found in " + this ) ) def mainClass = lib.pickOne( "Which one do you want to run?", classes.filter( lib.findMain(_).nonEmpty ) )( _.name.stripSuffix( "$" ) ) def classes = exportedClasspath.files.flatMap( lib.topLevelClasses( _, classLoader, false ) ) def flatClassLoader: Boolean = false override def classLoader: ClassLoader = taskCache[DependencyImplementation]( "classLoader" ).memoize{ if( flatClassLoader ){ new java.net.URLClassLoader(classpath.strings.map(f => new URL("file://" ++ f)).toArray) } else { /* if( concurrencyEnabled ){ // trigger concurrent building / downloading dependencies exportClasspathConcurrently } */ lib.classLoaderRecursion( this, (this +: transitiveDependencies).collect{ case d: ArtifactInfo => d }.groupBy( d => (d.groupId,d.artifactId) ).mapValues(_.head) ) } } // FIXME: these probably need to update outdated as well def classpath: ClassPath = exportedClasspath ++ dependencyClasspath def verifyClasspath: Unit = classpath.verify(lib) def dependencyClasspath: ClassPath = taskCache[DependencyImplementation]( "dependencyClasspath" ).memoize{ ClassPath( transitiveDependencies .flatMap(_.exportedClasspath.files) .distinct // <- currently needed here to handle diamond dependencies on builds (duplicate in classpath) ) } def dependencies: Seq[Dependency] /** return dependencies in order of linearized dependence. this is a bit tricky. */ def transitiveDependencies: Seq[Dependency] = taskCache[DependencyImplementation]( "transitiveDependencies" ).memoize{ lib.transitiveDependencies(this) } override def show: String = this.getClass.getSimpleName // ========== debug ========== def dependencyTree: String = lib.dependencyTreeRecursion(this) } // TODO: all this hard codes the scala version, needs more flexibility class ScalaCompilerDependency(cbtLastModified: Long, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef], classLoaderCache: ClassLoaderCache) extends BoundMavenDependency(cbtLastModified, mavenCache, MavenDependency("org.scala-lang","scala-compiler",version, Classifier.none), Seq(mavenCentral)) class ScalaLibraryDependency (cbtLastModified: Long, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef], classLoaderCache: ClassLoaderCache) extends BoundMavenDependency(cbtLastModified, mavenCache, MavenDependency("org.scala-lang","scala-library",version, Classifier.none), Seq(mavenCentral)) class ScalaReflectDependency (cbtLastModified: Long, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef], classLoaderCache: ClassLoaderCache) extends BoundMavenDependency(cbtLastModified, mavenCache, MavenDependency("org.scala-lang","scala-reflect",version, Classifier.none), Seq(mavenCentral)) class ScalaDependencies(cbtLastModified: Long, mavenCache: File, version: String)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef], classLoaderCache: ClassLoaderCache) extends Dependencies( Seq( new ScalaCompilerDependency(cbtLastModified, mavenCache, version), new ScalaLibraryDependency(cbtLastModified, mavenCache, version), new ScalaReflectDependency(cbtLastModified, mavenCache, version) ) ) case class BinaryDependency( paths: Seq[File], dependencies: Seq[Dependency] )(implicit val logger: Logger, val transientCache: java.util.Map[AnyRef,AnyRef], val classLoaderCache: ClassLoaderCache) extends DependencyImplementation{ assert(paths.nonEmpty) paths.foreach(p => assert(p.exists)) def exportedClasspath = ClassPath(paths) override def lastModified = paths.map(_.lastModifiedRecursive).max // FIXME: cache this def targetClasspath = exportedClasspath override lazy val moduleKey = this.getClass.getName + "(" + paths.mkString(", ") + ")" // PERFORMANCE HOTSPOT } /** Allows to easily assemble a bunch of dependencies */ case class Dependencies( dependencies: Seq[Dependency] )(implicit val logger: Logger, val transientCache: java.util.Map[AnyRef,AnyRef], val classLoaderCache: ClassLoaderCache) extends DependencyImplementation{ override def lastModified = dependencies.map(_.lastModified).maxOption.getOrElse(0) override lazy val moduleKey = this.getClass.getName + "(" + dependencies.map(_.moduleKey).mkString(", ") + ")" // PERFORMANCE HOTSPOT def targetClasspath = ClassPath() def exportedClasspath = ClassPath() override def show: String = this.getClass.getSimpleName + "( " + dependencies.map(_.show).mkString(", ") + " )" } case class PostBuildDependency(target: File, _dependencies: Seq[DependencyImplementation])(implicit val logger: Logger, val transientCache: java.util.Map[AnyRef,AnyRef], val classLoaderCache: ClassLoaderCache) extends DependencyImplementation{ override final lazy val lastModified = (target++".last-success").lastModified override lazy val moduleKey = target.string override def show = s"PostBuildDependency($target)" override def targetClasspath = exportedClasspath override def exportedClasspath = ClassPath( Seq(target) ) override def dependencies = _dependencies } case class CbtDependencies(cbtLastModified: Long, mavenCache: File, nailgunTarget: File, stage1Target: File, stage2Target: File, compatibilityTarget: File)(implicit logger: Logger, transientCache: java.util.Map[AnyRef,AnyRef], classLoaderCache: ClassLoaderCache){ val compatibilityDependency = PostBuildDependency(compatibilityTarget, Nil) val stage1Dependency = PostBuildDependency( stage1Target, Seq( PostBuildDependency(nailgunTarget, Nil), compatibilityDependency ) ++ MavenResolver(cbtLastModified,mavenCache,mavenCentral).bind( MavenDependency("org.scala-lang","scala-library",constants.scalaVersion), MavenDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,constants.scalaXmlVersion) ) ) val stage2Dependency = PostBuildDependency( stage2Target, stage1Dependency +: MavenResolver(cbtLastModified, mavenCache,mavenCentral).bind( MavenDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r"), MavenDependency("net.java.dev.jna", "jna-platform", "4.4.0"), MavenDependency("org.scala-lang","scala-compiler",constants.scalaVersion) ) ) } case class Classifier(name: Option[String]) object Classifier{ object none extends Classifier(None) object javadoc extends Classifier(Some("javadoc")) object sources extends Classifier(Some("sources")) } abstract class DependenciesProxy{ } case class MavenDependency( groupId: String, artifactId: String, version: String, classifier: Classifier = Classifier.none, verifyHash: Boolean = true ){ private[cbt] def serialize = // PERFORMANCE HOTSPOT groupId + ":" + artifactId + ":" + version + ( if(classifier.name.nonEmpty) ":" + classifier.name.get else "" ) private[cbt] def javafy: Array[String] = Array(groupId,artifactId,version) ++ classifier.name } object MavenDependency{ private[cbt] def deserialize = (_:String).split(":") match { case col => MavenDependency( col(0), col(1), col(2), Classifier(col.lift(3)) ) } private[cbt] def dejavafy = ( cols:Array[Array[String]] ) => cols.map( col => MavenDependency( col(0), col(1), col(2), Classifier(col.lift(3)) ) ).toSeq } // FIXME: take MavenResolver instead of mavenCache and repositories separately case class BoundMavenDependency( cbtLastModified: Long, mavenCache: File, mavenDependency: MavenDependency, repositories: Seq[URL], replace: MavenDependency => Seq[MavenDependency] => Seq[MavenDependency] = _ => (identity _) )( implicit val logger: Logger, val transientCache: java.util.Map[AnyRef,AnyRef], val classLoaderCache: ClassLoaderCache ) extends ArtifactInfo with DependencyImplementation{ override lazy val moduleKey = this.getClass.getName + "(" + mavenDependency.serialize + ")" // PERFORMANCE HOTSPOT override def hashCode = mavenDependency.hashCode override def equals(other: Any) = other match{ case o: BoundMavenDependency => o.mavenDependency == mavenDependency && o.repositories == repositories case _ => false } val MavenDependency( groupId, artifactId, version, classifier, verifyHash ) = mavenDependency assert( Option(groupId).collect{ case BoundMavenDependency.ValidIdentifier(_) => }.nonEmpty, s"not a valid groupId: '$groupId'" ) assert( Option(artifactId).collect{ case BoundMavenDependency.ValidIdentifier(_) => }.nonEmpty, s"not a valid artifactId: '$artifactId'" ) assert( version != "" && version != null && !version.startsWith(" ") && !version.endsWith(" "), s"not a valid version: '$version'" ) override def show: String = this.getClass.getSimpleName ++ "(" ++ mavenDependency.serialize ++ ")" override final lazy val lastModified: Long = taskCache[BoundMavenDependency]( "lastModified" ).memoize[java.lang.Long]{ classpath.strings.map(new File(_).lastModified).max } private lazy val base = { val v = BoundMavenDependency.extractVersion(version) "/" + groupId.split("\\.").mkString("/") + "/" + artifactId + "/" + v + "/" + artifactId + "-" + v } protected[cbt] def basePath(useClassifier: Boolean) = // PERFORMANCE HOTSPOT base + (if (useClassifier && classifier.name.nonEmpty) "-" + classifier.name.get else "") //private def coursierJarFile = userHome++"/.coursier/cache/v1/https/repo1.maven.org/maven2"++basePath++".jar" def exportedJars = Seq( jar ) override def exportedClasspath = ClassPath( exportedJars ) override def targetClasspath = exportedClasspath import scala.collection.JavaConverters._ private def resolve(suffix: String, hash: Option[String], useClassifier: Boolean): File = { logger.resolver(lib.blue("Resolving ")+this) val file = mavenCache ++ basePath(useClassifier) ++ "." ++ suffix val urls = repositories.map(_ ++ basePath(useClassifier) ++ "." ++ suffix) urls.find( lib.download(_, file, hash) ).getOrElse( throw new Exception(s"\nCannot resolve\n$this\nCan't find any of\n"++urls.mkString("\n")) ) file } private def resolveHash(suffix: String, useClassifier: Boolean) = { val path = resolve( suffix ++ ".sha1", None, useClassifier ).toPath Option( classLoaderCache.hashMap.get("hash:"+path) ).map(_.asInstanceOf[String]).getOrElse{ val result = Files.readAllLines( path, StandardCharsets.UTF_8 ).asScala.mkString("\n").split(" ").head.trim classLoaderCache.hashMap.put("hash:"+path, result) result } } def jarSha1: String = taskCache[BoundMavenDependency]("jarSha1").memoize{ resolveHash("jar", true) } def pomSha1: String = taskCache[BoundMavenDependency]("pomSha1").memoize{ resolveHash("pom", false) } def jar: File = taskCache[BoundMavenDependency]("jar").memoize{ resolve("jar", if(verifyHash) Some(jarSha1) else None, true) } def pom: File = taskCache[BoundMavenDependency]("pom").memoize{ resolve("pom", if(verifyHash) Some(pomSha1) else None, false) } private def pomXml = { logger.resolver( "Loading pom file: " ++ pom.string ) XML.loadFile(pom.string) } // ========== pom traversal ========== private lazy val transitivePom: Seq[BoundMavenDependency] = { (pomXml \ "parent").collect{ case parent => BoundMavenDependency( cbtLastModified: Long, mavenCache, MavenDependency( (parent \ "groupId").text, (parent \ "artifactId").text, (parent \ "version").text ), repositories, replace )(logger, transientCache, classLoaderCache) }.flatMap(_.transitivePom) :+ this } private lazy val properties: Map[String, String] = ( transitivePom.flatMap{ d => val props = (d.pomXml \ "properties").flatMap(_.child).map{ tag => tag.label -> tag.text } logger.pom(s"Found properties in $pom: $props") props } ).toMap private lazy val dependencyVersions: Map[String, (String,String)] = transitivePom.flatMap( p => (p.pomXml \ "dependencyManagement" \ "dependencies" \ "dependency").map{ xml => val groupId = p.lookup(xml,_ \ "groupId").get val artifactId = p.lookup(xml,_ \ "artifactId").get val version = p.lookup(xml,_ \ "version").get artifactId -> (groupId, version) } ).toMap def dependencies: Seq[BoundMavenDependency] = { replace(mavenDependency)( if(classifier == Classifier.sources) Seq() else { lib.cacheOnDisk( cbtLastModified, mavenCache ++ basePath(true) ++ ".pom.dependencies", classLoaderCache.hashMap )( MavenDependency.deserialize )( _.serialize )( MavenDependency.dejavafy )( _.map(_.javafy).toArray ){ (pomXml \ "dependencies" \ "dependency").collect{ case xml if ( (xml \ "scope").text == "" || (xml \ "scope").text == "compile" ) && (xml \ "optional").text != "true" => val artifactId = lookup(xml,_ \ "artifactId").get val groupId = lookup(xml,_ \ "groupId").getOrElse( dependencyVersions .get(artifactId).map(_._1) .getOrElse( throw new Exception(s"$artifactId not found in \n$dependencyVersions") ) ) val version = lookup(xml,_ \ "version").getOrElse( dependencyVersions .get(artifactId).map(_._2) .getOrElse( throw new Exception(s"$artifactId not found in \n$dependencyVersions") ) ) val classifier = Classifier( Some( (xml \ "classifier").text ).filterNot(_ == "").filterNot(_ == null) ) MavenDependency( groupId, artifactId, version, classifier ) }.toVector } } ).map( BoundMavenDependency( cbtLastModified, mavenCache, _, repositories, replace ) ).toVector } def lookup( xml: Node, accessor: Node => NodeSeq ): Option[String] = { // println("lookup in " + xml) val Substitution = "\\$\\{([^\\}]+)\\}".r def matcherFunc(matcher: scala.util.matching.Regex.Match): String = { val path = matcher.group(1) val ret = properties.get(path).orElse( transitivePom.reverse.flatMap{ d => Some(path.split("\\.").toList).collect{ case "project" :: path => path.foldLeft(d.pomXml:NodeSeq){ case (xml,tag) => xml \ tag }.text }.filter(_ != "") }.headOption ) .getOrElse( throw new Exception(s"Can't find $path in \n$properties.\n\npomParents: $transitivePom\n\n pomXml:\n$pomXml" ) ) Substitution.replaceAllIn( ret, matcherFunc _ ) } accessor(xml).headOption.map { v => Substitution.replaceAllIn( v.text, matcherFunc _ ) } } } object BoundMavenDependency{ private lazy val versionRangeRegex = "^[\\[\\(]([^,\\]\\)]*)(,([^\\]\\)]*))?[\\]\\)]".r def extractVersion( versionOrRange: String ) = { versionOrRange match { case versionRangeRegex(left,middle,right) if left != null || right != null => { Option(left).filterNot(_ == "") getOrElse right } case version => version } } val ValidIdentifier = "^([A-Za-z0-9_\\-.]+)$".r // according to maven's DefaultModelValidator.java def semanticVersionLessThan(left: Array[Either[Int,String]], right: Array[Either[Int,String]]) = { // FIXME: this ignores ends when different size val zipped = left zip right val res = zipped.map { case (Left(i),Left(j)) => i compare j case (Right(i),Right(j)) => i compare j case (Left(i),Right(j)) => i.toString compare j case (Right(i),Left(j)) => i compare j.toString } res.find(_ != 0).map(_ < 0).getOrElse(false) } def toInt(str: String): Either[Int,String] = try { Left(str.toInt) } catch { case e: NumberFormatException => Right(str) } /* this obviously should be overridable somehow */ def updateOutdated( deps: Seq[Dependency with ArtifactInfo], versionLessThan: (Array[Either[Int,String]], Array[Either[Int,String]]) => Boolean = semanticVersionLessThan )(implicit logger: Logger): Seq[Dependency with ArtifactInfo] = { val latest = deps .groupBy( d => (d.groupId, d.artifactId) ) .mapValues( _.groupBy(_.version) // remove duplicates .map( _._2.head ) .toVector .sortBy( _.version.split("\\.|\\-").map(toInt) )( Ordering.fromLessThan(versionLessThan) ) .last ) deps.map{ d => val l = latest((d.groupId,d.artifactId)) if(d != l) logger.resolver("outdated: "++d.show) l } } } /* case class DependencyOr(first: DirectoryDependency, second: JavaDependency) extends ProjectProxy with DirectoryDependencyBase{ val isFirst = new File(first.projectDirectory).exists protected val delegate = if(isFirst) first else second } */