path: root/stage1/resolver.scala

package cbt
import java.nio.file._
import java.nio.charset.StandardCharsets
import java.net._
import java.io._
import scala.xml._
import scala.concurrent._
import scala.concurrent.duration._

trait DependencyImplementation extends Dependency{
  implicit protected def logger: Logger
  protected def lib = new Stage1Lib(logger)

  /**
  CAREFUL: this is never allowed to return true for the same dependency more than 
  once in a single cbt run. Otherwise we can end up with multiple classloaders
  for the same classes since classLoaderRecursion recreates the classLoader when it
  sees this flag and it can be called multiple times. Maybe we can find a safer
  solution than this current state.
  */
  def needsUpdate: Boolean
  //def cacheClassLoader: Boolean = false
  private[cbt] def targetClasspath: ClassPath
  def dependencyClasspathArray: Array[File] = dependencyClasspath.files.toArray
  def exportedClasspathArray: Array[File] = exportedClasspath.files.toArray
  def exportedClasspath: ClassPath
  def dependenciesArray: Array[Dependency] = dependencies.to

  def needsUpdateCompat: java.lang.Boolean = needsUpdate

  /*
  //private type BuildCache = KeyLockedLazyCache[Dependency, Future[ClassPath]]
  def exportClasspathConcurrently: ClassPath = {
    // FIXME: this should separate a blocking and a non-blocking EC
    import scala.concurrent.ExecutionContext.Implicits.global
    Await.result(
      exportClasspathConcurrently(
        transitiveDependencies
          .collect{ case d: ArtifactInfo => d }
          .groupBy( d => (d.groupId,d.artifactId) )
          .mapValues( _.head )
        //, new BuildCache
      ), // FIXME
      Duration.Inf
    )
  }

  def concurrencyEnabled = false

  /**
  The implementation of this method is untested and likely buggy
  at this stage.
  */
  def exportClasspathConcurrently(
    latest: Map[(String, String),Dependency with ArtifactInfo]//, cache: BuildCache
  )( implicit ec: ExecutionContext ): Future[AnyRef] = {
    Future.sequence( // trigger compilation / download of all dependencies first
      this.dependencies.map{
        d =>
          // find out latest version of the required dependency
          val l = d match {
            case m: BoundMavenDependency => latest( (m.groupId,m.artifactId) )
            case _ => d
          }
          // // trigger compilation if not already triggered
          // cache.get( l, l.exportClasspathConcurrently( latest, cache ) )
          l.exportClasspathConcurrently( latest ) // FIXME
      }
    ).map(
      // merge dependency classpaths into one
      ClassPath.flatten(_)
    ).map(
      _ =>
      // now that all dependencies are done, compile the code of this
      exportedClasspath
    )
  }
  */

  def classLoader( cache: ClassLoaderCache ): ClassLoader = {
    /*
    if( concurrencyEnabled ){
      // trigger concurrent building / downloading dependencies
      exportClasspathConcurrently
    }
    */
    lib.classLoaderRecursion(
      this,
      (this +: transitiveDependencies).collect{
        case d: ArtifactInfo => d 
      }.groupBy(
        d => (d.groupId,d.artifactId)
      ).mapValues(_.head),
      cache // FIXME
    )
  }
  // FIXME: these probably need to update outdated as well
  def classpath           : ClassPath = exportedClasspath ++ dependencyClasspath
  def dependencyClasspath : ClassPath = ClassPath(
    transitiveDependencies
    .flatMap(_.exportedClasspath.files)
    .distinct // <- currently needed here to handle diamond dependencies on builds (duplicate in classpath)
  )
  def dependencies: Seq[Dependency]

  private object transitiveDependenciesCache extends Cache[Seq[Dependency]]
  /** return dependencies in order of linearized dependence. this is a bit tricky. */
  def transitiveDependencies: Seq[Dependency] = transitiveDependenciesCache{
    lib.transitiveDependencies(this)
  }

  override def show: String = this.getClass.getSimpleName
  // ========== debug ==========
  def dependencyTree: String = lib.dependencyTreeRecursion(this)
}

// TODO: all this hard codes the scala version, needs more flexibility
class ScalaCompilerDependency(cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-compiler",version, Classifier.none), Seq(mavenCentral))
class ScalaLibraryDependency (cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-library",version, Classifier.none), Seq(mavenCentral))
class ScalaReflectDependency (cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-reflect",version, Classifier.none), Seq(mavenCentral))

case class ScalaDependencies(cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit val logger: Logger) extends DependencyImplementation{ sd =>
  override final val needsUpdate = false
  def targetClasspath = ClassPath()
  def exportedClasspath = ClassPath()
  def dependencies = Seq(
    new ScalaCompilerDependency(cbtHasChanged, mavenCache, version),
    new ScalaLibraryDependency(cbtHasChanged, mavenCache, version),
    new ScalaReflectDependency(cbtHasChanged, mavenCache, version)
  )
}

case class BinaryDependency( path: File, dependencies: Seq[Dependency] )(implicit val logger: Logger) extends DependencyImplementation{
  def exportedClasspath = ClassPath(Seq(path))
  override def needsUpdate = false
  def targetClasspath = exportedClasspath
}

/** Allows to easily assemble a bunch of dependencies */
case class Dependencies( dependencies: Seq[Dependency] )(implicit val logger: Logger) extends DependencyImplementation{
  override def needsUpdate = dependencies.exists(_.needsUpdate)
  override def exportedClasspath = ClassPath()
  override def targetClasspath = ClassPath()
}

case class Stage1Dependency(cbtHasChanged: Boolean, mavenCache: File, nailgunTarget: File, stage1Target: File, compatibilityTarget: File)(implicit val logger: Logger) extends DependencyImplementation{
  override def needsUpdate = false
  override def targetClasspath = exportedClasspath
  override def exportedClasspath = ClassPath( Seq(nailgunTarget, stage1Target) )
  val compatibilityDependency = CompatibilityDependency(cbtHasChanged, compatibilityTarget)
  override def dependencies = Seq(
    compatibilityDependency
  ) ++
  MavenResolver(cbtHasChanged,mavenCache,mavenCentral).bind(
    MavenDependency("org.scala-lang","scala-library",constants.scalaVersion),
    MavenDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,constants.scalaXmlVersion)
  )
}
case class CompatibilityDependency(cbtHasChanged: Boolean, compatibilityTarget: File)(implicit val logger: Logger) extends DependencyImplementation{
  override def needsUpdate = false
  override def targetClasspath = exportedClasspath
  override def exportedClasspath = ClassPath( Seq(compatibilityTarget) )
  override def dependencies = Seq()
}
case class CbtDependency(cbtHasChanged: Boolean, mavenCache: File, nailgunTarget: File, stage1Target: File, stage2Target: File, compatibilityTarget: File)(implicit val logger: Logger) extends DependencyImplementation{
  override def needsUpdate = false
  override def targetClasspath = exportedClasspath
  override def exportedClasspath = ClassPath( Seq( stage2Target ) )
  val stage1Dependency = Stage1Dependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, compatibilityTarget)
  override def dependencies = Seq(
    stage1Dependency
  ) ++ 
  MavenResolver(cbtHasChanged, mavenCache,mavenCentral).bind(
    MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0"),
    MavenDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r")
  )
}

case class Classifier(name: Option[String])
object Classifier{
  object none extends Classifier(None)
  object javadoc extends Classifier(Some("javadoc"))
  object sources extends Classifier(Some("sources"))
}
abstract class DependenciesProxy{

}
class BoundMavenDependencies(
  cbtHasChanged: Boolean, mavenCache: File, urls: Seq[URL], mavenDependencies: Seq[MavenDependency]
)(implicit logger: Logger) extends Dependencies(
  mavenDependencies.map( BoundMavenDependency(cbtHasChanged,mavenCache,_,urls) )
)
case class MavenDependency(
  groupId: String, artifactId: String, version: String, classifier: Classifier = Classifier.none
){
  private[cbt] def serialize = groupId ++ ":" ++ artifactId ++ ":"++ version ++ classifier.name.map(":" ++ _).getOrElse("")
}
object MavenDependency{
  private[cbt] def deserialize = (_:String).split(":") match {
    case col => MavenDependency( col(0), col(1), col(2), Classifier(col.lift(3)) )
  }
}
// FIXME: take MavenResolver instead of mavenCache and repositories separately
case class BoundMavenDependency(
  cbtHasChanged: Boolean, mavenCache: File, mavenDependency: MavenDependency, repositories: Seq[URL]
)(implicit val logger: Logger) extends ArtifactInfo with DependencyImplementation{
  val MavenDependency( groupId, artifactId, version, classifier ) = mavenDependency
  assert(
    Option(groupId).collect{
      case BoundMavenDependency.ValidIdentifier(_) =>
    }.nonEmpty,
    s"not a valid groupId: '$groupId'"
  )
  assert(
    Option(artifactId).collect{
      case BoundMavenDependency.ValidIdentifier(_) =>
    }.nonEmpty,
    s"not a valid artifactId: '$artifactId'"
  )
  assert(
    version != "" && version != null && !version.startsWith(" ") && !version.endsWith(" "),
    s"not a valid version: '$version'"
  )
    override def show: String = this.getClass.getSimpleName ++ "(" ++ mavenDependency.serialize ++ ")"

  override def needsUpdate = false

  private val groupPath = groupId.split("\\.").mkString("/")
  protected[cbt] def basePath = s"/$groupPath/$artifactId/$version/$artifactId-$version" ++ classifier.name.map("-"++_).getOrElse("")
  
  //private def coursierJarFile = userHome++"/.coursier/cache/v1/https/repo1.maven.org/maven2"++basePath++".jar"

  def exportedJars = Seq( jar )
  override def exportedClasspath = ClassPath( exportedJars )
  override def targetClasspath = exportedClasspath
  import scala.collection.JavaConversions._

  private def resolve(suffix: String, hash: Option[String]): File = {
    logger.resolver("Resolving "+this)
    val file = mavenCache ++ basePath ++ "." ++ suffix
    val urls = repositories.map(_ ++ basePath ++ "." ++ suffix)
    urls.find(
      lib.download(_, file, hash)
    ).getOrElse(
      throw new Exception(s"\nCannot resolve\n$this\nCan't find any of\n"++urls.mkString("\n"))
    )
    file
  }

  private def resolveHash(suffix: String) = {
    Files.readAllLines(
      resolve( suffix ++ ".sha1", None ).toPath,
      StandardCharsets.UTF_8
    ).mkString("\n").split(" ").head.trim
  }
  
  private object jarSha1Cache extends Cache[String]
  def jarSha1: String = jarSha1Cache{ resolveHash("jar") }

  private object pomSha1Cache extends Cache[String]
  def pomSha1: String = pomSha1Cache{ resolveHash("pom") }

  private object jarCache extends Cache[File]
  def jar: File = jarCache{ resolve("jar", Some(jarSha1)) }

  private object pomCache extends Cache[File]
  def pom: File = pomCache{ resolve("pom", Some(pomSha1)) }

  private def pomXml = XML.loadFile(pom.string)
  // ========== pom traversal ==========

  private lazy val transitivePom: Seq[BoundMavenDependency] = {
    (pomXml \ "parent").collect{
      case parent =>
        BoundMavenDependency(
          cbtHasChanged: Boolean,
          mavenCache,
          MavenDependency(
            (parent \ "groupId").text,
            (parent \ "artifactId").text,
            (parent \ "version").text
          ),
          repositories
        )(logger)
    }.flatMap(_.transitivePom) :+ this
  }

  private lazy val properties: Map[String, String] = (
    transitivePom.flatMap{ d =>
      val props = (d.pomXml \ "properties").flatMap(_.child).map{
        tag => tag.label -> tag.text
      }
      logger.pom(s"Found properties in $pom: $props")
      props
    }
  ).toMap

  private lazy val dependencyVersions: Map[String, (String,String)] =
    transitivePom.flatMap(
      p =>
      (p.pomXml \ "dependencyManagement" \ "dependencies" \ "dependency").map{
        xml =>
          val groupId = p.lookup(xml,_ \ "groupId").get
          val artifactId = p.lookup(xml,_ \ "artifactId").get
          val version = p.lookup(xml,_ \ "version").get
          artifactId -> (groupId, version)
      }
    ).toMap

  def dependencies: Seq[BoundMavenDependency] = {
    if(classifier == Classifier.sources) Seq()
    else {
      lib.cacheOnDisk(
        cbtHasChanged, mavenCache ++ basePath ++ ".pom.dependencies"
      )( MavenDependency.deserialize )( _.serialize ){
        (pomXml \ "dependencies" \ "dependency").collect{
          case xml if ( (xml \ "scope").text == "" || (xml \ "scope").text == "compile" ) && (xml \ "optional").text != "true" =>
            val artifactId = lookup(xml,_ \ "artifactId").get
            val groupId =
              lookup(xml,_ \ "groupId").getOrElse(
                dependencyVersions
                  .get(artifactId).map(_._1)
                  .getOrElse(
                    throw new Exception(s"$artifactId not found in \n$dependencyVersions")
                  )
              )
            val version =
              lookup(xml,_ \ "version").getOrElse(
                dependencyVersions
                  .get(artifactId).map(_._2)
                  .getOrElse(
                    throw new Exception(s"$artifactId not found in \n$dependencyVersions")
                  )
              )
            val classifier = Classifier( Some( (xml \ "classifier").text ).filterNot(_ == "").filterNot(_ == null) )
            MavenDependency( groupId, artifactId, version, classifier )
        }.toVector
      }.map(
        BoundMavenDependency( cbtHasChanged, mavenCache, _, repositories )
      ).to
    }
  }
  def lookup( xml: Node, accessor: Node => NodeSeq ): Option[String] = {
    //println("lookup in "++pomUrl)
    val Substitution = "\\$\\{([^\\}]+)\\}".r
    accessor(xml).headOption.map{v =>
      //println("found: "++v.text)
      Substitution.replaceAllIn(
        v.text,
        matcher => {
          val path = matcher.group(1)
          properties.get(path).orElse(
            transitivePom.reverse.flatMap{ d =>
              Some(path.split("\\.").toList).collect{
                case "project" :: path =>
                  path.foldLeft(d.pomXml:NodeSeq){ case (xml,tag) => xml \ tag }.text
              }.filter(_ != "")
            }.headOption
          )
          .getOrElse(
            throw new Exception(s"Can't find $path in \n$properties.\n\npomParents: $transitivePom\n\n pomXml:\n$pomXml" )
          )
            //println("lookup "++path ++ ": "++(pomXml\path).text)          
        }
      )
    }
  }
}
object BoundMavenDependency{
  def ValidIdentifier = "^([A-Za-z0-9_\\-.]+)$".r // according to maven's DefaultModelValidator.java
  def semanticVersionLessThan(left: Array[Either[Int,String]], right: Array[Either[Int,String]]) = {
    // FIXME: this ignores ends when different size
    val zipped = left zip right
    val res = zipped.map {
      case (Left(i),Left(j)) => i compare j
      case (Right(i),Right(j)) => i compare j
      case (Left(i),Right(j)) => i.toString compare j
      case (Right(i),Left(j)) => i compare j.toString
    }
    res.find(_ != 0).map(_ < 0).getOrElse(false)
  }
  def toInt(str: String): Either[Int,String] = try {
    Left(str.toInt)
  } catch {
    case e: NumberFormatException => Right(str)
  }
  /* this obviously should be overridable somehow */
  def updateOutdated(
    deps: Seq[Dependency with ArtifactInfo],
    versionLessThan: (Array[Either[Int,String]], Array[Either[Int,String]]) => Boolean = semanticVersionLessThan
  )(implicit logger: Logger): Seq[Dependency with ArtifactInfo] = {
    val latest = deps
      .groupBy( d => (d.groupId, d.artifactId) )
      .mapValues(
        _.groupBy(_.version) // remove duplicates
         .map( _._2.head )
         .toVector
         .sortBy( _.version.split("\\.|\\-").map(toInt) )( Ordering.fromLessThan(versionLessThan) )
         .last
      )
    deps.map{
      d => 
        val l = latest((d.groupId,d.artifactId))
        if(d != l) logger.resolver("outdated: "++d.show)
        l
    }
  }
}