Fix incremental compilation when a Scala project depends on a Java project (#414)

* Upgrade ammonite to 1.1.2-30-53edc31

This is mainly to get https://github.com/lihaoyi/Ammonite/pull/851 which
should reduce the amount of unnecessary work done by incremental
compilation in the Mill build. This requires some code changes since
this means we now depend on a more recent version of coursier, as a
side-effect this means that we do not depend on scalaz anymore.

Also use the same ammonite version in the Mill build and in
ScalaModule#ammoniteReplClasspath.

Also remove an incorrect dependency in the caffeine integration test.
This was always wrong but did not start failing until this commit,
probably due to dependencies appearing in a different order on the
classpath.

* Rename ScalaWorker to ZincWorker

Starting with the next commit, it will be used in Java-only projects
too, so the name is misleading.

* Upgrade to Zinc 1.2.1

* Fix incremental compilation when a Scala project depends on a Java project

Before this commit, JavaModule#compile simply called javac
unconditionally, thus generating new classfiles every time. But if a
Scala project depends on a Java project, this will throw off the
incremental compilation algorithm which will unnecessarily recompile
files. To avoid this we now use Zinc to compile Java projects too (as a
bonus this means that Java compilation becomes incremental). This
required some refactoring in ZincWorkerImpl to be able to compile stuff
without having to pass Scala-specific options.

The issue solved by this commit could be reproduced by running in the
Mill repository:

$ mill main.compile
$ mill -i
@ main.compile()

and observing that before this commit, the `main.compile()` call ended
up recompiling code.

7 files changed, 58 insertions, 85 deletions

diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala
index 259e5cd3..ab4f8795 100644
--- a/scalalib/src/mill/scalalib/JavaModule.scala
+++ b/scalalib/src/mill/scalalib/JavaModule.scala
@@ -17,12 +17,13 @@ import mill.util.Loose.Agg
   * Core configuration required to compile a single Scala compilation target
   */
 trait JavaModule extends mill.Module with TaskModule { outer =>
-  def scalaWorker: ScalaWorkerModule = mill.scalalib.ScalaWorkerModule
+  def zincWorker: ZincWorkerModule = mill.scalalib.ZincWorkerModule
 
   trait Tests extends TestModule{
     override def moduleDeps = Seq(outer)
     override def repositories = outer.repositories
     override def javacOptions = outer.javacOptions
+    override def zincWorker = outer.zincWorker
   }
   def defaultCommandName() = "run"
 
@@ -38,7 +39,16 @@ trait JavaModule extends mill.Module with TaskModule { outer =>
   def finalMainClassOpt: T[Either[String, String]] = T{
     mainClass() match{
       case Some(m) => Right(m)
-      case None => Left("No main class specified or found")
+      case None =>
+        zincWorker.worker().discoverMainClasses(compile())match {
+          case Seq() => Left("No main class specified or found")
+          case Seq(main) => Right(main)
+          case mains =>
+            Left(
+              s"Multiple main classes found (${mains.mkString(",")}) " +
+                "please explicitly specify which one to use by overriding mainClass"
+            )
+        }
     }
   }
 
@@ -98,7 +108,7 @@ trait JavaModule extends mill.Module with TaskModule { outer =>
   }
 
 
-  def repositories: Seq[Repository] = scalaWorker.repositories
+  def repositories: Seq[Repository] = zincWorker.repositories
 
   def platformSuffix = T{ "" }
 
@@ -133,12 +143,12 @@ trait JavaModule extends mill.Module with TaskModule { outer =>
     } yield PathRef(path)
   }
 
-  def compile: T[CompilationResult] = T{
-    Lib.compileJava(
-      allSourceFiles().map(_.path.toIO).toArray,
-      compileClasspath().map(_.path.toIO).toArray,
-      javacOptions(),
-      upstreamCompileOutput()
+  def compile: T[CompilationResult] = T.persistent{
+    zincWorker.worker().compileJava(
+      upstreamCompileOutput(),
+      allSourceFiles().map(_.path),
+      compileClasspath().map(_.path),
+      javacOptions()
     )
   }
 
@@ -317,7 +327,7 @@ trait JavaModule extends mill.Module with TaskModule { outer =>
     val (procId, procTombstone, token) = backgroundSetup(T.ctx().dest)
     try Result.Success(Jvm.interactiveSubprocess(
       "mill.scalalib.backgroundwrapper.BackgroundWrapper",
-      (runClasspath() ++ scalaWorker.backgroundWrapperClasspath()).map(_.path),
+      (runClasspath() ++ zincWorker.backgroundWrapperClasspath()).map(_.path),
       forkArgs(),
       forkEnv(),
       Seq(procId.toString, procTombstone.toString, token, finalMainClass()) ++ args,
@@ -332,7 +342,7 @@ trait JavaModule extends mill.Module with TaskModule { outer =>
     val (procId, procTombstone, token) = backgroundSetup(T.ctx().dest)
     try Result.Success(Jvm.interactiveSubprocess(
       "mill.scalalib.backgroundwrapper.BackgroundWrapper",
-      (runClasspath() ++ scalaWorker.backgroundWrapperClasspath()).map(_.path),
+      (runClasspath() ++ zincWorker.backgroundWrapperClasspath()).map(_.path),
       forkArgs(),
       forkEnv(),
       Seq(procId.toString, procTombstone.toString, token, mainClass) ++ args,
@@ -384,7 +394,7 @@ trait TestModule extends JavaModule with TaskModule {
 
     Jvm.subprocess(
       mainClass = "mill.scalalib.TestRunner",
-      classPath = scalaWorker.scalalibClasspath().map(_.path),
+      classPath = zincWorker.scalalibClasspath().map(_.path),
       jvmArgs = forkArgs(),
       envArgs = forkEnv(),
       mainArgs =
diff --git a/scalalib/src/mill/scalalib/Lib.scala b/scalalib/src/mill/scalalib/Lib.scala
index e890caaf..827c6bf4 100644
--- a/scalalib/src/mill/scalalib/Lib.scala
+++ b/scalalib/src/mill/scalalib/Lib.scala
@@ -27,34 +27,6 @@ object CompilationResult {
 case class CompilationResult(analysisFile: Path, classes: PathRef)
 
 object Lib{
-  def compileJava(sources: Array[java.io.File],
-                  classpath: Array[java.io.File],
-                  javaOpts: Seq[String],
-                  upstreamCompileOutput: Seq[CompilationResult])
-                 (implicit ctx: mill.util.Ctx) = {
-    val javac = ToolProvider.getSystemJavaCompiler()
-    if (javac == null) {
-      throw new Exception(
-        "Your Java installation is not a JDK, so it can't compile Java code;" +
-        " Please install the JDK version of Java")
-    }
-
-    rm(ctx.dest / 'classes)
-    mkdir(ctx.dest / 'classes)
-    val cpArgs =
-      if(classpath.isEmpty) Seq()
-      else Seq("-cp", classpath.mkString(File.pathSeparator))
-
-    val args = Seq("-d", ctx.dest / 'classes) ++ cpArgs ++ javaOpts ++ sources
-
-    javac.run(
-      ctx.log.inStream, ctx.log.outputStream, ctx.log.errorStream,
-      args.map(_.toString):_*
-    )
-    if (ls(ctx.dest / 'classes).isEmpty) mill.eval.Result.Failure("Compilation Failed")
-    else mill.eval.Result.Success(CompilationResult(ctx.dest / 'zinc, PathRef(ctx.dest / 'classes)))
-  }
-
   private val ReleaseVersion = raw"""(\d+)\.(\d+)\.(\d+)""".r
   private val MinorSnapshotVersion = raw"""(\d+)\.(\d+)\.([1-9]\d*)-SNAPSHOT""".r
   private val DottyVersion = raw"""0\.(\d+)\.(\d+).*""".r
@@ -104,7 +76,7 @@ object Lib{
   /**
     * Resolve dependencies using Coursier.
     *
-    * We do not bother breaking this out into the separate ScalaWorker classpath,
+    * We do not bother breaking this out into the separate ZincWorker classpath,
     * because Coursier is already bundled with mill/Ammonite to support the
     * `import $ivy` syntax.
     */
diff --git a/scalalib/src/mill/scalalib/ScalaModule.scala b/scalalib/src/mill/scalalib/ScalaModule.scala
index 1899ee14..3e90521e 100644
--- a/scalalib/src/mill/scalalib/ScalaModule.scala
+++ b/scalalib/src/mill/scalalib/ScalaModule.scala
@@ -24,7 +24,7 @@ trait ScalaModule extends JavaModule { outer =>
     override def scalacPluginIvyDeps = outer.scalacPluginIvyDeps
     override def scalacOptions = outer.scalacOptions
     override def javacOptions = outer.javacOptions
-    override def scalaWorker = outer.scalaWorker
+    override def zincWorker = outer.zincWorker
     override def moduleDeps: Seq[JavaModule] = Seq(outer)
   }
 
@@ -60,29 +60,10 @@ trait ScalaModule extends JavaModule { outer =>
     )
   }
 
-  override def finalMainClassOpt: T[Either[String, String]] = T{
-    mainClass() match{
-      case Some(m) => Right(m)
-      case None =>
-        scalaWorker.worker().discoverMainClasses(compile())match {
-          case Seq() => Left("No main class specified or found")
-          case Seq(main) => Right(main)
-          case mains =>
-            Left(
-              s"Multiple main classes found (${mains.mkString(",")}) " +
-                "please explicitly specify which one to use by overriding mainClass"
-            )
-        }
-    }
-  }
-
-
   def scalacPluginIvyDeps = T{ Agg.empty[Dep] }
 
   def scalacOptions = T{ Seq.empty[String] }
 
-  override def repositories: Seq[Repository] = scalaWorker.repositories
-
   private val Milestone213 = raw"""2.13.(\d+)-M(\d+)""".r
 
   def scalaCompilerBridgeSources = T {
@@ -142,16 +123,16 @@ trait ScalaModule extends JavaModule { outer =>
   }
 
   override def compile: T[CompilationResult] = T.persistent{
-    scalaWorker.worker().compileScala(
-      scalaVersion(),
+    zincWorker.worker().compileMixed(
+      upstreamCompileOutput(),
       allSourceFiles().map(_.path),
-      scalaCompilerBridgeSources(),
       compileClasspath().map(_.path),
-      scalaCompilerClasspath().map(_.path),
+      javacOptions(),
+      scalaVersion(),
       scalacOptions(),
+      scalaCompilerBridgeSources(),
+      scalaCompilerClasspath().map(_.path),
       scalacPluginClasspath().map(_.path),
-      javacOptions(),
-      upstreamCompileOutput()
     )
   }
 
@@ -204,7 +185,7 @@ trait ScalaModule extends JavaModule { outer =>
     unmanagedClasspath() ++
     resolveDeps(T.task{
       runIvyDeps() ++ scalaLibraryIvyDeps() ++ transitiveIvyDeps() ++
-      Agg(ivy"com.lihaoyi:::ammonite:1.1.2")
+      Agg(ivy"com.lihaoyi:::ammonite:${Versions.ammonite}")
     })()
   }
 
diff --git a/scalalib/src/mill/scalalib/Versions.scala b/scalalib/src/mill/scalalib/Versions.scala
index 6c1c385f..bfcc5495 100644
--- a/scalalib/src/mill/scalalib/Versions.scala
+++ b/scalalib/src/mill/scalalib/Versions.scala
@@ -1,6 +1,8 @@
 package mill.scalalib
 
 object Versions {
+  // Keep synchronized with ammonite dependency in core in build.sc
+  val ammonite = "1.1.2-30-53edc31"
   // Keep synchronized with zinc dependency in scalalib.worker in build.sc
-  val zinc = "1.1.7"
+  val zinc = "1.2.1"
 }
diff --git a/scalalib/src/mill/scalalib/ScalaWorkerApi.scala b/scalalib/src/mill/scalalib/ZincWorkerApi.scala
index 547cc5db..ebe2f071 100644
--- a/scalalib/src/mill/scalalib/ScalaWorkerApi.scala
+++ b/scalalib/src/mill/scalalib/ZincWorkerApi.scala
@@ -11,10 +11,10 @@ import mill.scalalib.Lib.resolveDependencies
 import mill.util.Loose
 import mill.util.JsonFormatters._
 
-object ScalaWorkerModule extends mill.define.ExternalModule with ScalaWorkerModule{
+object ZincWorkerModule extends mill.define.ExternalModule with ZincWorkerModule{
   lazy val millDiscover = Discover[this.type]
 }
-trait ScalaWorkerModule extends mill.Module{
+trait ZincWorkerModule extends mill.Module{
   def repositories = Seq(
     Cache.ivy2Local,
     MavenRepository("https://repo1.maven.org/maven2"),
@@ -36,15 +36,15 @@ trait ScalaWorkerModule extends mill.Module{
     )
   }
 
-  def worker: Worker[ScalaWorkerApi] = T.worker{
+  def worker: Worker[ZincWorkerApi] = T.worker{
     val cl = mill.util.ClassLoader.create(
       classpath().map(_.path.toNIO.toUri.toURL).toVector,
       getClass.getClassLoader
     )
-    val cls = cl.loadClass("mill.scalalib.worker.ScalaWorkerImpl")
+    val cls = cl.loadClass("mill.scalalib.worker.ZincWorkerImpl")
     val instance = cls.getConstructor(classOf[mill.util.Ctx], classOf[Array[String]])
       .newInstance(T.ctx(), compilerInterfaceClasspath().map(_.path.toString).toArray[String])
-    instance.asInstanceOf[ScalaWorkerApi]
+    instance.asInstanceOf[ZincWorkerApi]
   }
 
   def compilerInterfaceClasspath = T{
@@ -57,20 +57,26 @@ trait ScalaWorkerModule extends mill.Module{
 
 }
 
-trait ScalaWorkerApi {
+trait ZincWorkerApi {
+  /** Compile a Java-only project */
+  def compileJava(upstreamCompileOutput: Seq[CompilationResult],
+                  sources: Agg[Path],
+                  compileClasspath: Agg[Path],
+                  javacOptions: Seq[String])
+                 (implicit ctx: mill.util.Ctx): mill.eval.Result[CompilationResult]
 
-  def compileScala(scalaVersion: String,
+  /** Compile a mixed Scala/Java or Scala-only project */
+  def compileMixed(upstreamCompileOutput: Seq[CompilationResult],
                    sources: Agg[Path],
-                   compilerBridgeSources: Path,
                    compileClasspath: Agg[Path],
-                   compilerClasspath: Agg[Path],
-                   scalacOptions: Seq[String],
-                   scalacPluginClasspath: Agg[Path],
                    javacOptions: Seq[String],
-                   upstreamCompileOutput: Seq[CompilationResult])
+                   scalaVersion: String,
+                   scalacOptions: Seq[String],
+                   compilerBridgeSources: Path,
+                   compilerClasspath: Agg[Path],
+                   scalacPluginClasspath: Agg[Path])
                   (implicit ctx: mill.util.Ctx): mill.eval.Result[CompilationResult]
 
-
   def discoverMainClasses(compilationResult: CompilationResult)
                          (implicit ctx: mill.util.Ctx): Seq[String]
 }
diff --git a/scalalib/src/mill/scalalib/dependency/metadata/MavenMetadataLoader.scala b/scalalib/src/mill/scalalib/dependency/metadata/MavenMetadataLoader.scala
index 7f25764b..491911bf 100644
--- a/scalalib/src/mill/scalalib/dependency/metadata/MavenMetadataLoader.scala
+++ b/scalalib/src/mill/scalalib/dependency/metadata/MavenMetadataLoader.scala
@@ -2,16 +2,18 @@ package mill.scalalib.dependency.metadata
 
 import coursier.Cache
 import coursier.maven.MavenRepository
+import coursier.util.Task
 import mill.scalalib.dependency.versions.Version
 
 private[dependency] final case class MavenMetadataLoader(mavenRepo: MavenRepository)
     extends MetadataLoader {
 
-  private val fetch = Cache.fetch()
+  private val fetch = Cache.fetch[Task]()
 
   override def getVersions(module: coursier.Module): List[Version] = {
+    import scala.concurrent.ExecutionContext.Implicits.global
     // TODO fallback to 'versionsFromListing' if 'versions' doesn't work? (needs to be made public in coursier first)
-    val allVersions = mavenRepo.versions(module, fetch).run.unsafePerformSync
+    val allVersions = mavenRepo.versions(module, fetch).run.unsafeRun
     allVersions
       .map(_.available.map(Version(_)))
       .getOrElse(List.empty)
diff --git a/scalalib/src/mill/scalalib/scalafmt/ScalafmtModule.scala b/scalalib/src/mill/scalalib/scalafmt/ScalafmtModule.scala
index c4d64b0e..74f64e28 100644
--- a/scalalib/src/mill/scalalib/scalafmt/ScalafmtModule.scala
+++ b/scalalib/src/mill/scalalib/scalafmt/ScalafmtModule.scala
@@ -23,7 +23,7 @@ trait ScalafmtModule extends JavaModule {
 
   def scalafmtDeps: T[Agg[PathRef]] = T {
     Lib.resolveDependencies(
-      scalaWorker.repositories,
+      zincWorker.repositories,
       Lib.depToDependency(_, "2.12.4"),
       Seq(ivy"com.geirsson::scalafmt-cli:${scalafmtVersion()}")
     )