Fix incremental compilation when a Scala project depends on a Java project (#414)

* Upgrade ammonite to 1.1.2-30-53edc31

This is mainly to get https://github.com/lihaoyi/Ammonite/pull/851 which
should reduce the amount of unnecessary work done by incremental
compilation in the Mill build. This requires some code changes since
this means we now depend on a more recent version of coursier, as a
side-effect this means that we do not depend on scalaz anymore.

Also use the same ammonite version in the Mill build and in
ScalaModule#ammoniteReplClasspath.

Also remove an incorrect dependency in the caffeine integration test.
This was always wrong but did not start failing until this commit,
probably due to dependencies appearing in a different order on the
classpath.

* Rename ScalaWorker to ZincWorker

Starting with the next commit, it will be used in Java-only projects
too, so the name is misleading.

* Upgrade to Zinc 1.2.1

* Fix incremental compilation when a Scala project depends on a Java project

Before this commit, JavaModule#compile simply called javac
unconditionally, thus generating new classfiles every time. But if a
Scala project depends on a Java project, this will throw off the
incremental compilation algorithm which will unnecessarily recompile
files. To avoid this we now use Zinc to compile Java projects too (as a
bonus this means that Java compilation becomes incremental). This
required some refactoring in ZincWorkerImpl to be able to compile stuff
without having to pass Scala-specific options.

The issue solved by this commit could be reproduced by running in the
Mill repository:

$ mill main.compile
$ mill -i
@ main.compile()

and observing that before this commit, the `main.compile()` call ended
up recompiling code.

1 files changed, 92 insertions, 41 deletions

diff --git a/scalalib/worker/src/mill/scalalib/worker/ScalaWorkerImpl.scala b/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala
index ebeb9561..6b21de84 100644
--- a/scalalib/worker/src/mill/scalalib/worker/ScalaWorkerImpl.scala
+++ b/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala
@@ -24,9 +24,31 @@ case class MockedLookup(am: File => Optional[CompileAnalysis]) extends PerClassp
     Locate.definesClass(classpathEntry)
 }
 
-class ScalaWorkerImpl(ctx0: mill.util.Ctx,
-                      compilerBridgeClasspath: Array[String]) extends mill.scalalib.ScalaWorkerApi{
-  @volatile var compilersCache = Option.empty[(Long, Compilers)]
+class ZincWorkerImpl(ctx0: mill.util.Ctx,
+                     compilerBridgeClasspath: Array[String]) extends mill.scalalib.ZincWorkerApi{
+  private val ic = new sbt.internal.inc.IncrementalCompilerImpl()
+  val javaOnlyCompilers = {
+    // Keep the classpath as written by the user
+    val classpathOptions = ClasspathOptions.of(false, false, false, false, false)
+
+    val dummyFile = new java.io.File("")
+    // Zinc does not have an entry point for Java-only compilation, so we need
+    // to make up a dummy ScalaCompiler instance.
+    val scalac = ZincUtil.scalaCompiler(
+      new ScalaInstance("", null, null, dummyFile, dummyFile, new Array(0), Some("")), null,
+      classpathOptions // this is used for javac too
+    )
+
+    ic.compilers(
+      instance = null,
+      classpathOptions,
+      None,
+      scalac
+    )
+  }
+
+  @volatile var mixedCompilersCache = Option.empty[(Long, Compilers)]
+
 
   /** Compile the bridge if it doesn't exist yet and return the output directory.
    *  TODO: Proper invalidation, see #389
@@ -78,27 +100,38 @@ class ScalaWorkerImpl(ctx0: mill.util.Ctx,
       .getOrElse(Seq.empty[String])
   }
 
+  def compileJava(upstreamCompileOutput: Seq[CompilationResult],
+                  sources: Agg[Path],
+                  compileClasspath: Agg[Path],
+                  javacOptions: Seq[String])
+                 (implicit ctx: mill.util.Ctx): mill.eval.Result[CompilationResult] = {
+    compileInternal(
+      upstreamCompileOutput,
+      sources,
+      compileClasspath,
+      javacOptions,
+      scalacOptions = Nil,
+      javaOnlyCompilers
+    )
+  }
 
-  def compileScala(scalaVersion: String,
+  def compileMixed(upstreamCompileOutput: Seq[CompilationResult],
                    sources: Agg[Path],
-                   compilerBridgeSources: Path,
                    compileClasspath: Agg[Path],
-                   compilerClasspath: Agg[Path],
-                   scalacOptions: Seq[String],
-                   scalacPluginClasspath: Agg[Path],
                    javacOptions: Seq[String],
-                   upstreamCompileOutput: Seq[CompilationResult])
+                   scalaVersion: String,
+                   scalacOptions: Seq[String],
+                   compilerBridgeSources: Path,
+                   compilerClasspath: Agg[Path],
+                   scalacPluginClasspath: Agg[Path])
                   (implicit ctx: mill.util.Ctx): mill.eval.Result[CompilationResult] = {
-    val compileClasspathFiles = compileClasspath.map(_.toIO).toArray
     val compilerJars = compilerClasspath.toArray.map(_.toIO)
 
     val compilerBridge = compileZincBridgeIfNeeded(scalaVersion, compilerBridgeSources, compilerJars)
-
-    val ic = new sbt.internal.inc.IncrementalCompilerImpl()
-
     val compilerBridgeSig = compilerBridge.mtime.toMillis
+
     val compilersSig = compilerBridgeSig + compilerClasspath.map(p => p.toString().hashCode + p.mtime.toMillis).sum
-    val compilers = compilersCache match {
+    val compilers = mixedCompilersCache match {
       case Some((k, v)) if k == compilersSig => v
       case _ =>
         val compilerName =
@@ -120,12 +153,28 @@ class ScalaWorkerImpl(ctx0: mill.util.Ctx,
           None,
           ZincUtil.scalaCompiler(scalaInstance, compilerBridge.toIO)
         )
-        compilersCache = Some((compilersSig, compilers))
+        mixedCompilersCache = Some((compilersSig, compilers))
         compilers
     }
 
-    mkdir(ctx.dest)
+    compileInternal(
+      upstreamCompileOutput,
+      sources,
+      compileClasspath,
+      javacOptions,
+      scalacOptions = scalacPluginClasspath.map(jar => s"-Xplugin:${jar}").toSeq ++ scalacOptions,
+      compilers
+    )
+  }
 
+  private def compileInternal(upstreamCompileOutput: Seq[CompilationResult],
+                              sources: Agg[Path],
+                              compileClasspath: Agg[Path],
+                              javacOptions: Seq[String],
+                              scalacOptions: Seq[String],
+                              compilers: Compilers)
+                             (implicit ctx: mill.util.Ctx): mill.eval.Result[CompilationResult] = {
+    mkdir(ctx.dest)
 
     val logger = {
       val consoleAppender = MainAppender.defaultScreen(ConsoleOut.printStreamOut(
@@ -158,33 +207,35 @@ class ScalaWorkerImpl(ctx0: mill.util.Ctx,
 
     val store = FileAnalysisStore.binary(zincIOFile)
 
+    val inputs = ic.inputs(
+      classpath = classesIODir +: compileClasspath.map(_.toIO).toArray,
+      sources = sources.toArray.map(_.toIO),
+      classesDirectory = classesIODir,
+      scalacOptions = scalacOptions.toArray,
+      javacOptions = javacOptions.toArray,
+      maxErrors = 10,
+      sourcePositionMappers = Array(),
+      order = CompileOrder.Mixed,
+      compilers = compilers,
+      setup = ic.setup(
+        lookup,
+        skip = false,
+        zincIOFile,
+        new FreshCompilerCache,
+        IncOptions.of(),
+        new ManagedLoggedReporter(10, logger),
+        None,
+        Array()
+      ),
+      pr = {
+        val prev = store.get()
+        PreviousResult.of(prev.map(_.getAnalysis), prev.map(_.getMiniSetup))
+      }
+    )
+
     try {
       val newResult = ic.compile(
-        ic.inputs(
-          classpath = classesIODir +: compileClasspathFiles,
-          sources = sources.toArray.map(_.toIO),
-          classesDirectory = classesIODir,
-          scalacOptions = (scalacPluginClasspath.map(jar => s"-Xplugin:${jar}") ++ scalacOptions).toArray,
-          javacOptions = javacOptions.toArray,
-          maxErrors = 10,
-          sourcePositionMappers = Array(),
-          order = CompileOrder.Mixed,
-          compilers = compilers,
-          setup = ic.setup(
-            lookup,
-            skip = false,
-            zincIOFile,
-            new FreshCompilerCache,
-            IncOptions.of(),
-            new ManagedLoggedReporter(10, logger),
-            None,
-            Array()
-          ),
-          pr = {
-            val prev = store.get()
-            PreviousResult.of(prev.map(_.getAnalysis), prev.map(_.getMiniSetup))
-          }
-        ),
+        in = inputs,
         logger = logger
       )