Implement alternative resolvers

This commit also
- requires Dependencies to explicitly implement canBeCached
- unifies some logic for dependency downloading
- moves SBT-like dependency DSL into its own trait
- error message showing build directory for exceptions in builds
Not splitting this up in favor of faster progress to 1.0.

The user facing API can probably be slightly improved using implicits
inside of BasicBuild, but we can do that later.

1 files changed, 85 insertions, 66 deletions

diff --git a/stage1/resolver.scala b/stage1/resolver.scala
index 701871c..ea20b89 100644
--- a/stage1/resolver.scala
+++ b/stage1/resolver.scala
@@ -35,7 +35,7 @@ abstract class Dependency{
   def exportedJars: Seq[File]
   def jars: Seq[File] = exportedJars ++ dependencyJars
 
-  def canBeCached = false
+  def canBeCached: Boolean
 
   //private type BuildCache = KeyLockedLazyCache[Dependency, Future[ClassPath]]
   def exportClasspathConcurrently: ClassPath = {
@@ -68,7 +68,7 @@ abstract class Dependency{
         d =>
           // find out latest version of the required dependency
           val l = d match {
-            case m: JavaDependency => latest( (m.groupId,m.artifactId) )
+            case m: BoundMavenDependency => latest( (m.groupId,m.artifactId) )
             case _ => d
           }
           // // trigger compilation if not already triggered
@@ -166,7 +166,7 @@ abstract class Dependency{
       case _:ArtifactInfo => false
       case _ => true
     }
-    noInfo ++ JavaDependency.updateOutdated( hasInfo ).reverse.distinct
+    noInfo ++ BoundMavenDependency.updateOutdated( hasInfo ).reverse.distinct
   }
 
   def show: String = this.getClass.getSimpleName
@@ -182,9 +182,9 @@ abstract class Dependency{
 }
 
 // TODO: all this hard codes the scala version, needs more flexibility
-class ScalaCompilerDependency(version: String)(implicit logger: Logger) extends JavaDependency("org.scala-lang","scala-compiler",version)
-class ScalaLibraryDependency (version: String)(implicit logger: Logger) extends JavaDependency("org.scala-lang","scala-library",version)
-class ScalaReflectDependency (version: String)(implicit logger: Logger) extends JavaDependency("org.scala-lang","scala-reflect",version)
+class ScalaCompilerDependency(version: String)(implicit logger: Logger) extends BoundMavenDependency(MavenDependency("org.scala-lang","scala-compiler",version, Classifier.none), Seq(MavenRepository.central.url))
+class ScalaLibraryDependency (version: String)(implicit logger: Logger) extends BoundMavenDependency(MavenDependency("org.scala-lang","scala-library",version, Classifier.none), Seq(MavenRepository.central.url))
+class ScalaReflectDependency (version: String)(implicit logger: Logger) extends BoundMavenDependency(MavenDependency("org.scala-lang","scala-reflect",version, Classifier.none), Seq(MavenRepository.central.url))
 
 case class ScalaDependencies(version: String)(implicit val logger: Logger) extends Dependency{ sd =>
   override final val needsUpdate = false
@@ -199,7 +199,7 @@ case class ScalaDependencies(version: String)(implicit val logger: Logger) exten
   )
 }
 
-case class BinaryDependency( path: File, dependencies: Seq[Dependency] )(implicit val logger: Logger) extends Dependency{
+case class BinaryDependency( path: File, dependencies: Seq[Dependency], canBeCached: Boolean )(implicit val logger: Logger) extends Dependency{
   def exportedClasspath = ClassPath(Seq(path))
   def exportedJars = Seq[File](path)
   override def needsUpdate = false
@@ -207,16 +207,19 @@ case class BinaryDependency( path: File, dependencies: Seq[Dependency] )(implici
 }
 
 /** Allows to easily assemble a bunch of dependencies */
-case class Dependencies( _dependencies: Dependency* )(implicit val logger: Logger) extends Dependency{
-  override def dependencies = _dependencies.to
-  def needsUpdate = dependencies.exists(_.needsUpdate)
-  def exportedClasspath = ClassPath(Seq())
-  def exportedJars = Seq()
-  def targetClasspath = ClassPath(Seq())
+case class Dependencies( dependencies: Seq[Dependency] )(implicit val logger: Logger) extends Dependency{
+  override def needsUpdate = dependencies.exists(_.needsUpdate)
+  override def canBeCached = dependencies.forall(_.canBeCached)
+  override def exportedClasspath = ClassPath(Seq())
+  override def exportedJars = Seq()
+  override def targetClasspath = ClassPath(Seq())
+}
+object Dependencies{
+  def apply( dependencies: Dependency* )(implicit logger: Logger): Dependencies = Dependencies( dependencies.to )
 }
 
 case class Stage1Dependency()(implicit val logger: Logger) extends Dependency{
-  def needsUpdate = false // FIXME: think this through, might allow simplifications and/or optimizations
+  override def needsUpdate = false // FIXME: think this through, might allow simplifications and/or optimizations
   override def canBeCached = false
   /*
   private object classLoaderRecursionCache extends Cache[ClassLoader]
@@ -232,23 +235,27 @@ case class Stage1Dependency()(implicit val logger: Logger) extends Dependency{
   override def exportedClasspath = ClassPath( Seq(nailgunTarget, stage1Target) )
   override def exportedJars = ???//Seq[File]()  
   override def dependencies = Seq(
-    JavaDependency("org.scala-lang","scala-library",constants.scalaVersion),
-    JavaDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,"1.0.5")
+    MavenRepository.central.resolve(
+      MavenDependency("org.scala-lang","scala-library",constants.scalaVersion),
+      MavenDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,"1.0.5")
+    )
   )
   // FIXME: implement sanity check to prevent using incompatible scala-library and xml version on cp
   override def classLoaderRecursion( latest: Map[(String,String),Dependency], cache: ClassLoaderCache )
     = getClass.getClassLoader
 }
 case class CbtDependency()(implicit val logger: Logger) extends Dependency{
-  def needsUpdate = false // FIXME: think this through, might allow simplifications and/or optimizations
+  override def needsUpdate = false // FIXME: think this through, might allow simplifications and/or optimizations
   override def canBeCached = false
   override def targetClasspath = exportedClasspath
   override def exportedClasspath = ClassPath( Seq( stage2Target ) )
   override def exportedJars = ???
   override def dependencies = Seq(
     Stage1Dependency(),
-    JavaDependency("net.incongru.watchservice","barbary-watchservice","1.0"),
-    JavaDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r")
+    MavenRepository.central.resolve(
+      MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0"),
+      MavenDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r")
+    )
   )
 }
 
@@ -258,19 +265,30 @@ object Classifier{
   object javadoc extends Classifier(Some("javadoc"))
   object sources extends Classifier(Some("sources"))
 }
+abstract class DependenciesProxy{
 
-case class JavaDependency(
+}
+class BoundMavenDependencies(
+  urls: Seq[URL], mavenDependencies: Seq[MavenDependency]
+)(implicit logger: Logger) extends Dependencies(
+  mavenDependencies.map( BoundMavenDependency(_,urls) )
+)
+case class MavenDependency(
   groupId: String, artifactId: String, version: String, classifier: Classifier = Classifier.none
+)
+case class BoundMavenDependency(
+  mavenDependency: MavenDependency, repositories: Seq[URL]
 )(implicit val logger: Logger) extends ArtifactInfo{
+  val MavenDependency( groupId, artifactId, version, classifier ) = mavenDependency
   assert(
     Option(groupId).collect{
-      case JavaDependency.ValidIdentifier(_) =>
+      case BoundMavenDependency.ValidIdentifier(_) =>
     }.nonEmpty,
     s"not a valid groupId: '$groupId'"
   )
   assert(
     Option(artifactId).collect{
-      case JavaDependency.ValidIdentifier(_) =>
+      case BoundMavenDependency.ValidIdentifier(_) =>
     }.nonEmpty,
     s"not a valid artifactId: '$artifactId'"
   )
@@ -280,62 +298,60 @@ case class JavaDependency(
   )
 
   override def needsUpdate = false
-  override def canBeCached = true
+  override def canBeCached = dependencies.forall(_.canBeCached)
 
   private val groupPath = groupId.split("\\.").mkString("/")
-  def basePath = s"/$groupPath/$artifactId/$version/$artifactId-$version" ++ classifier.name.map("-"++_).getOrElse("")
+  protected[cbt] def basePath = s"/$groupPath/$artifactId/$version/$artifactId-$version" ++ classifier.name.map("-"++_).getOrElse("")
   
-  private def resolverUrl:URL = new URL(
-    if(version.endsWith("-SNAPSHOT")) "https://oss.sonatype.org/content/repositories/snapshots" else "https://repo1.maven.org/maven2"
-  )
-  private def baseUrl: URL = resolverUrl ++ basePath
-  private def baseFile: File = mavenCache ++ basePath
-  private def pomFile: File = baseFile ++ ".pom"
-  private def jarFile: File = baseFile ++ ".jar"
   //private def coursierJarFile = userHome++"/.coursier/cache/v1/https/repo1.maven.org/maven2"++basePath++".jar"
-  private def pomUrl: URL = baseUrl ++ ".pom"
-  private[cbt] def jarUrl: URL = baseUrl ++ ".jar"    
 
-  def exportedJars = Seq( jar )
-  def exportedClasspath = ClassPath( exportedJars )
-  def targetClasspath = exportedClasspath
+  override def exportedJars = Seq( jar )
+  override def exportedClasspath = ClassPath( exportedJars )
+  override def targetClasspath = exportedClasspath
   import scala.collection.JavaConversions._
-  
-  def jarSha1 = {
-    val file = jarFile ++ ".sha1"
-    lib.download( jarUrl ++ ".sha1"  , file, None )
-    // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c  jna-3.2.2.pom
-    Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim
+
+  private def resolve(suffix: String, hash: Option[String]): File = {
+    val file = mavenCache ++ basePath ++ "." ++ suffix
+    val urls = repositories.map(_ ++ basePath ++ "." ++ suffix)
+    urls.find(
+      lib.download(_, file, hash)
+    ).getOrElse(
+      throw new Exception(s"\nCannot resolve\n$this\nCan't find any of\n"++urls.mkString("\n"))
+    )
+    file
   }
 
-  def pomSha1 = {
-    val file = pomFile++".sha1"
-    lib.download( pomUrl++".sha1" , file, None )
-    // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c  jna-3.2.2.pom
-    Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim
+  private def resolveHash(suffix: String) = {
+    Files.readAllLines(
+      resolve( suffix ++ ".sha1", None ).toPath
+    ).mkString("\n").split(" ").head.trim
   }
+  
+  private object jarSha1Cache extends Cache[String]
+  def jarSha1: String = jarSha1Cache{ resolveHash("jar") }
+
+  private object pomSha1Cache extends Cache[String]
+  def pomSha1: String = pomSha1Cache{ resolveHash("pom") }
 
   private object jarCache extends Cache[File]
-  def jar = jarCache{
-    lib.download( jarUrl, jarFile, Some(jarSha1) )
-    jarFile
-  }
-  def pomXml = XML.loadFile(pom.toString)
+  def jar: File = jarCache{ resolve("jar", Some(jarSha1)) }
 
-  def pom = {
-    lib.download( pomUrl, pomFile, Some(pomSha1) )
-    pomFile
-  }
+  private object pomCache extends Cache[File]
+  def pom: File = pomCache{ resolve("pom", Some(pomSha1)) }
 
+  def pomXml = XML.loadFile(pom.string)
   // ========== pom traversal ==========
 
-  lazy val transitivePom: Seq[JavaDependency] = {
+  lazy val transitivePom: Seq[BoundMavenDependency] = {
     (pomXml \ "parent").collect{
       case parent =>
-        JavaDependency(
-          (parent \ "groupId").text,
-          (parent \ "artifactId").text,
-          (parent \ "version").text
+        BoundMavenDependency(
+          MavenDependency(
+            (parent \ "groupId").text,
+            (parent \ "artifactId").text,
+            (parent \ "version").text
+          ),
+          repositories
         )(logger)
     }.flatMap(_.transitivePom) :+ this
   }
@@ -362,7 +378,7 @@ case class JavaDependency(
       }
     ).toMap
 
-  def dependencies: Seq[JavaDependency] = {
+  def dependencies: Seq[BoundMavenDependency] = {
     if(classifier == Classifier.sources) Seq()
     else (pomXml \ "dependencies" \ "dependency").collect{
       case xml if (xml \ "scope").text == "" && (xml \ "optional").text != "true" =>
@@ -383,9 +399,12 @@ case class JavaDependency(
                 throw new Exception(s"$artifactId not found in \n$dependencyVersions")
               )
           )
-        JavaDependency(
-          groupId, artifactId, version,
-          Classifier( Some( (xml \ "classifier").text ).filterNot(_ == "").filterNot(_ == null) )
+        BoundMavenDependency(
+          MavenDependency(
+            groupId, artifactId, version,
+            Classifier( Some( (xml \ "classifier").text ).filterNot(_ == "").filterNot(_ == null) )
+          ),
+          repositories
         )
     }.toVector
   }
@@ -411,7 +430,7 @@ case class JavaDependency(
     }
   }
 }
-object JavaDependency{
+object BoundMavenDependency{
   def ValidIdentifier = "^([A-Za-z0-9_\\-.]+)$".r // according to maven's DefaultModelValidator.java
   def semanticVersionLessThan(left: String, right: String) = {
     // FIXME: this ignores ends when different size