Use the binary version of the compiler bridge when available (#659)

* Use the binary version of the compiler bridge when available, fix #591

This also eliminates #389 for Dotty 0.13.0-RC1 and more recent

* Add test for Dotty 0.16

5 files changed, 113 insertions, 62 deletions

diff --git a/scalalib/api/src/ZincWorkerApi.scala b/scalalib/api/src/ZincWorkerApi.scala
index 790ea274..0480e8e6 100644
--- a/scalalib/api/src/ZincWorkerApi.scala
+++ b/scalalib/api/src/ZincWorkerApi.scala
@@ -45,7 +45,7 @@ object CompilationResult {
 // analysisFile is represented by os.Path, so we won't break caches after file changes
 case class CompilationResult(analysisFile: os.Path, classes: PathRef)
 
-object Util{
+object Util {
   def isDotty(scalaVersion: String) =
     scalaVersion.startsWith("0.")
 
@@ -66,16 +66,22 @@ object Util{
   private val ReleaseVersion = raw"""(\d+)\.(\d+)\.(\d+)""".r
   private val MinorSnapshotVersion = raw"""(\d+)\.(\d+)\.([1-9]\d*)-SNAPSHOT""".r
   private val DottyVersion = raw"""0\.(\d+)\.(\d+).*""".r
+  private val DottyNightlyVersion = raw"""0\.(\d+)\.(\d+)-bin-(.*)-NIGHTLY""".r
   private val TypelevelVersion = raw"""(\d+)\.(\d+)\.(\d+)-bin-typelevel.*""".r
 
 
-  def scalaBinaryVersion(scalaVersion: String) = {
-    scalaVersion match {
+  def scalaBinaryVersion(scalaVersion: String) = scalaVersion match {
       case ReleaseVersion(major, minor, _) => s"$major.$minor"
       case MinorSnapshotVersion(major, minor, _) => s"$major.$minor"
       case DottyVersion(minor, _) => s"0.$minor"
       case TypelevelVersion(major, minor, _) => s"$major.$minor"
       case _ => scalaVersion
-    }
   }
-}
\ No newline at end of file
+
+  /** @return true if the compiler bridge can be downloaded as an already compiled jar */
+  def isBinaryBridgeAvailable(scalaVersion: String) = scalaVersion match {
+      case DottyNightlyVersion(minor, _, _) => minor.toInt >= 14 // 0.14.0-bin or more (not 0.13.0-bin)
+      case DottyVersion(minor, _) => minor.toInt >= 13 // 0.13.0-RC1 or more
+      case _ => false
+  }
+}
diff --git a/scalalib/src/ZincWorkerModule.scala b/scalalib/src/ZincWorkerModule.scala
index 4c94102c..81bfbb46 100644
--- a/scalalib/src/ZincWorkerModule.scala
+++ b/scalalib/src/ZincWorkerModule.scala
@@ -7,15 +7,15 @@ import mill.T
 import mill.api.KeyedLockedCache
 import mill.define.{Discover, Worker}
 import mill.scalalib.Lib.resolveDependencies
-import mill.scalalib.api.Util.isDotty
+import mill.scalalib.api.Util.{isDotty, isBinaryBridgeAvailable}
 import mill.scalalib.api.ZincWorkerApi
 import mill.api.Loose
 import mill.util.JsonFormatters._
 
-object ZincWorkerModule extends mill.define.ExternalModule with ZincWorkerModule{
+object ZincWorkerModule extends mill.define.ExternalModule with ZincWorkerModule {
   lazy val millDiscover = Discover[this.type]
 }
-trait ZincWorkerModule extends mill.Module{
+trait ZincWorkerModule extends mill.Module {
   def repositories = Seq(
     coursier.LocalRepositories.ivy2Local,
     MavenRepository("https://repo1.maven.org/maven2"),
@@ -37,7 +37,8 @@ trait ZincWorkerModule extends mill.Module{
     )
   }
 
-  def worker: Worker[mill.scalalib.api.ZincWorkerApi] = T.worker{
+  def worker: Worker[mill.scalalib.api.ZincWorkerApi] = T.worker {
+    val cp = compilerInterfaceClasspath()
     val cl = mill.api.ClassLoader.create(
       classpath().map(_.path.toNIO.toUri.toURL).toVector,
       getClass.getClassLoader
@@ -46,7 +47,7 @@ trait ZincWorkerModule extends mill.Module{
     val instance = cls.getConstructor(
       classOf[
         Either[
-          (ZincWorkerApi.Ctx, Array[os.Path], (String, String) => os.Path),
+          (ZincWorkerApi.Ctx, (String, String) => (Option[Array[os.Path]], os.Path)),
           String => os.Path
         ]
       ],
@@ -58,8 +59,7 @@ trait ZincWorkerModule extends mill.Module{
       .newInstance(
         Left((
           T.ctx(),
-          compilerInterfaceClasspath().map(_.path).toArray,
-          (x: String, y: String) => scalaCompilerBridgeSourceJar(x, y).asSuccess.get.value
+          (x: String, y: String) => scalaCompilerBridgeJar(x, y, cp).asSuccess.get.value
         )),
         mill.scalalib.api.Util.grepJar(_, "scala-library", _, sources = false),
         mill.scalalib.api.Util.grepJar(_, "scala-compiler", _, sources = false),
@@ -69,8 +69,9 @@ trait ZincWorkerModule extends mill.Module{
     instance.asInstanceOf[mill.scalalib.api.ZincWorkerApi]
   }
 
-  def scalaCompilerBridgeSourceJar(scalaVersion: String,
-                                   scalaOrganization: String) = {
+  def scalaCompilerBridgeJar(scalaVersion: String,
+                             scalaOrganization: String,
+                             compileClassPath: Agg[mill.eval.PathRef]) = {
     val (scalaVersion0, scalaBinaryVersion0) = scalaVersion match {
       case s if s.startsWith("2.13.") => ("2.13.0-M2", "2.13.0-M2")
       case _ => (scalaVersion, mill.scalalib.api.Util.scalaBinaryVersion(scalaVersion))
@@ -88,15 +89,18 @@ trait ZincWorkerModule extends mill.Module{
         val version = Versions.zinc
         (ivy"$org::$name:$version", s"${name}_$scalaBinaryVersion0", version)
       }
+    val useSources = !isBinaryBridgeAvailable(scalaVersion)
 
     resolveDependencies(
       repositories,
-      Lib.depToDependency(_, scalaVersion0, ""),
+      Lib.depToDependency(_, scalaVersion0),
       Seq(bridgeDep),
-      sources = true
-    ).map(deps =>
-      mill.scalalib.api.Util.grepJar(deps.map(_.path), bridgeName, bridgeVersion, sources = true)
-    )
+      useSources
+    ).map{deps =>
+      val cp = if (useSources) Some(compileClassPath.map(_.path).toArray) else None
+      val res = mill.scalalib.api.Util.grepJar(deps.map(_.path), bridgeName, bridgeVersion, useSources)
+      (cp, res)
+    }
   }
 
   def compilerInterfaceClasspath = T{
diff --git a/scalalib/test/resources/hello-dotty/boo/src/Main.scala b/scalalib/test/resources/hello-dotty/boo/src/Main.scala
new file mode 100644
index 00000000..6b19f294
--- /dev/null
+++ b/scalalib/test/resources/hello-dotty/boo/src/Main.scala
@@ -0,0 +1,17 @@
+import cats._, cats.data._, cats.implicits._
+
+trait Context
+
+object Main {
+  def foo(f: given Int => Int): Int = {
+    given x as Int = 1
+    f
+  }
+
+  def main(args: Array[String]): Unit = {
+    val x = Applicative[List].pure(1)
+    assert(x == List(1))
+    val value = foo(given x => x + 1)
+    assert(value == 2)
+  }
+}
diff --git a/scalalib/test/src/HelloWorldTests.scala b/scalalib/test/src/HelloWorldTests.scala
index d74d7d64..957709eb 100644
--- a/scalalib/test/src/HelloWorldTests.scala
+++ b/scalalib/test/src/HelloWorldTests.scala
@@ -264,6 +264,10 @@ object HelloWorldTests extends TestSuite {
       def scalaVersion = "0.9.0-RC1"
       def ivyDeps = Agg(ivy"org.typelevel::cats-core:1.2.0".withDottyCompat(scalaVersion()))
      }
+    object boo extends ScalaModule {
+      def scalaVersion = "0.16.0-RC3"
+      def ivyDeps = Agg(ivy"org.typelevel::cats-core:1.6.1".withDottyCompat(scalaVersion()))
+    }
   }
 
   val resourcePath = os.pwd / 'scalalib / 'test / 'resources / "hello-world"
diff --git a/scalalib/worker/src/ZincWorkerImpl.scala b/scalalib/worker/src/ZincWorkerImpl.scala
index 4d3de464..cf37812c 100644
--- a/scalalib/worker/src/ZincWorkerImpl.scala
+++ b/scalalib/worker/src/ZincWorkerImpl.scala
@@ -22,7 +22,7 @@ case class MockedLookup(am: File => Optional[CompileAnalysis]) extends PerClassp
 }
 
 class ZincWorkerImpl(compilerBridge: Either[
-                       (ZincWorkerApi.Ctx, Array[os.Path], (String, String) => os.Path),
+                       (ZincWorkerApi.Ctx, (String, String) => (Option[Array[os.Path]], os.Path)),
                        String => os.Path
                      ],
                      libraryJarNameGrep: (Agg[os.Path], String) => os.Path,
@@ -68,55 +68,75 @@ class ZincWorkerImpl(compilerBridge: Either[
       scaladocMethod.invoke(scaladocClass.newInstance(), args.toArray).asInstanceOf[Boolean]
     }
   }
-  /** Compile the bridge if it doesn't exist yet and return the output directory.
-    *  TODO: Proper invalidation, see #389
+
+  /** Compile the SBT/Zinc compiler bridge in the `compileDest` directory */
+  def compileZincBridge(ctx0: ZincWorkerApi.Ctx,
+                        workingDir: os.Path,
+                        compileDest: os.Path,
+                        scalaVersion: String,
+                        compilerJars: Array[File],
+                        compilerBridgeClasspath: Array[os.Path],
+                        compilerBridgeSourcesJar: os.Path): Unit = {
+    ctx0.log.info("Compiling compiler interface...")
+
+    os.makeDir.all(workingDir)
+    os.makeDir.all(compileDest)
+
+    val sourceFolder = mill.api.IO.unpackZip(compilerBridgeSourcesJar)(workingDir)
+    val classloader = mill.api.ClassLoader.create(compilerJars.map(_.toURI.toURL), null)(ctx0)
+
+    val sources = os.walk(sourceFolder.path).filter(a => a.ext == "scala" || a.ext == "java")
+
+    val argsArray = Array[String](
+      "-d", compileDest.toString,
+      "-classpath", (compilerJars ++ compilerBridgeClasspath).mkString(File.pathSeparator)
+    ) ++ sources.map(_.toString)
+
+    val allScala = sources.forall(_.ext == "scala")
+    val allJava = sources.forall(_.ext == "java")
+    if (allJava) {
+      import scala.sys.process._
+      (Seq("javac") ++ argsArray).!
+    } else if (allScala) {
+      val compilerMain = classloader.loadClass(
+        if (isDotty(scalaVersion)) "dotty.tools.dotc.Main"
+        else "scala.tools.nsc.Main"
+      )
+      compilerMain
+        .getMethod("process", classOf[Array[String]])
+        .invoke(null, argsArray)
+    } else {
+      throw new IllegalArgumentException("Currently not implemented case.")
+    }
+  }
+
+  /** If needed, compile (for Scala 2) or download (for Dotty) the compiler bridge.
+    * @return a path to the directory containing the compiled classes, or to the downloaded jar file
     */
-  def compileZincBridgeIfNeeded(scalaVersion: String, scalaOrganization: String, compilerJars: Array[File]): os.Path = {
-    compilerBridge match{
+  def compileBridgeIfNeeded(scalaVersion: String, scalaOrganization: String, compilerClasspath: Agg[os.Path]): os.Path = {
+    compilerBridge match {
       case Right(compiled) => compiled(scalaVersion)
-      case Left((ctx0, compilerBridgeClasspath, srcJars)) =>
+      case Left((ctx0, bridgeProvider)) =>
         val workingDir = ctx0.dest / scalaVersion
         val compiledDest = workingDir / 'compiled
-        if (!os.exists(workingDir)) {
-          ctx0.log.info("Compiling compiler interface...")
-
-          os.makeDir.all(workingDir)
-          os.makeDir.all(compiledDest)
-
-          val sourceFolder = mill.api.IO.unpackZip(srcJars(scalaVersion, scalaOrganization))(workingDir)
-          val classloader = mill.api.ClassLoader.create(compilerJars.map(_.toURI.toURL), null)(ctx0)
-
-          val sources = os.walk(sourceFolder.path).filter(a => a.ext == "scala" || a.ext == "java")
-
-          val argsArray = Array[String](
-            "-d", compiledDest.toString,
-            "-classpath", (compilerJars ++ compilerBridgeClasspath).mkString(File.pathSeparator)
-          ) ++ sources.map(_.toString)
-
-          val allScala = sources.forall(_.ext == "scala")
-          val allJava = sources.forall(_.ext == "java")
-          if (allJava) {
-            import scala.sys.process._
-            (Seq("javac") ++ argsArray).!
-          } else if (allScala) {
-            val compilerMain = classloader.loadClass(
-              if (isDotty(scalaVersion)) "dotty.tools.dotc.Main"
-              else "scala.tools.nsc.Main"
-            )
-            compilerMain.getMethod("process", classOf[Array[String]])
-              .invoke(null, argsArray)
-          } else {
-            throw new IllegalArgumentException("Currently not implemented case.")
-          }
 
+        if (os.exists(compiledDest)) {
+          compiledDest
+        } else {
+          val (cp, bridgeJar) = bridgeProvider(scalaVersion, scalaOrganization)
+          cp match {
+            case None =>
+              bridgeJar
+            case Some(bridgeClasspath) =>
+              val compilerJars = compilerClasspath.toArray.map(_.toIO)
+              compileZincBridge(ctx0, workingDir, compiledDest, scalaVersion, compilerJars, bridgeClasspath, bridgeJar)
+              compiledDest
+          }
         }
-        compiledDest
     }
 
   }
 
-
-
   def discoverMainClasses(compilationResult: CompilationResult)
                          (implicit ctx: ZincWorkerApi.Ctx): Seq[String] = {
     def toScala[A](o: Optional[A]): Option[A] = if (o.isPresent) Some(o.get) else None
@@ -238,10 +258,10 @@ class ZincWorkerImpl(compilerBridge: Either[
     val combinedCompilerClasspath = compilerClasspath ++ scalacPluginClasspath
     val combinedCompilerJars = combinedCompilerClasspath.toArray.map(_.toIO)
 
-    val compiledCompilerBridge = compileZincBridgeIfNeeded(
+    val compiledCompilerBridge = compileBridgeIfNeeded(
       scalaVersion,
       scalaOrganization,
-      compilerClasspath.toArray.map(_.toIO)
+      compilerClasspath
     )
 
     val compilerBridgeSig = os.mtime(compiledCompilerBridge)
@@ -305,7 +325,7 @@ class ZincWorkerImpl(compilerBridge: Either[
     val lookup = MockedLookup(analysisMap)
 
     val zincFile = ctx.dest / 'zinc
-    val classesDir = 
+    val classesDir =
       if (compileToJar) ctx.dest / "classes.jar"
       else ctx.dest / "classes"