Generalize Zinc Worker (#514)

* Generalize Zinc worker

- Compiler bridges can now be either pre-compiled or on-demand-compiled
- Scala library/compiler jar discovery is now configurable
- Zinc compiler cache is now configurable, rather than being hardcoded at n=1

* .

* update constructor args

* remove duplicate util/AggWrapper.scala file

* fix

* fix

* fix

* cleanup

8 files changed, 66 insertions, 42 deletions

diff --git a/scalalib/src/GenIdeaImpl.scala b/scalalib/src/GenIdeaImpl.scala
index 548b8d4e..b8f9d35e 100644
--- a/scalalib/src/GenIdeaImpl.scala
+++ b/scalalib/src/GenIdeaImpl.scala
@@ -5,8 +5,8 @@ import coursier.{Cache, CoursierPaths, Repository}
 import mill.define._
 import mill.eval.{Evaluator, PathRef, Result}
 import mill.api.Ctx.{Home, Log}
-import mill.util.Strict.Agg
-import mill.util.{Loose, Strict}
+import mill.api.Strict.Agg
+import mill.api.{Loose, Strict}
 import mill.{T, scalalib}
 
 import scala.util.Try
diff --git a/scalalib/src/JavaModule.scala b/scalalib/src/JavaModule.scala
index 78be8893..72c0a5a6 100644
--- a/scalalib/src/JavaModule.scala
+++ b/scalalib/src/JavaModule.scala
@@ -9,7 +9,7 @@ import mill.modules.{Assembly, Jvm}
 import mill.modules.Jvm.{createAssembly, createJar}
 import Lib._
 import mill.scalalib.publish.{Artifact, Scope}
-import mill.util.Loose.Agg
+import mill.api.Loose.Agg
 
 /**
   * Core configuration required to compile a single Scala compilation target
diff --git a/scalalib/src/MiscModule.scala b/scalalib/src/MiscModule.scala
index c6449d6e..bf64f1f3 100644
--- a/scalalib/src/MiscModule.scala
+++ b/scalalib/src/MiscModule.scala
@@ -4,7 +4,7 @@ package scalalib
 import mill.define.Cross.Resolver
 import mill.define.{Cross, Task}
 import mill.eval.{PathRef, Result}
-import mill.util.Loose.Agg
+import mill.api.Loose.Agg
 object CrossModuleBase{
   def scalaVersionPaths(scalaVersion: String, f: String => os.Path) = {
     for(segments <- scalaVersion.split('.').inits.filter(_.nonEmpty))
diff --git a/scalalib/src/ScalaModule.scala b/scalalib/src/ScalaModule.scala
index 9d669bf4..5fad1664 100644
--- a/scalalib/src/ScalaModule.scala
+++ b/scalalib/src/ScalaModule.scala
@@ -8,7 +8,7 @@ import mill.modules.Jvm
 import mill.modules.Jvm.createJar
 import mill.scalalib.api.Util.isDotty
 import Lib._
-import mill.util.Loose.Agg
+import mill.api.Loose.Agg
 import mill.api.DummyInputStream
 
 /**
@@ -79,36 +79,7 @@ trait ScalaModule extends JavaModule { outer =>
 
   def scalaDocOptions = T{ scalacOptions() }
 
-  private val Milestone213 = raw"""2.13.(\d+)-M(\d+)""".r
 
-  def scalaCompilerBridgeSources = T {
-    val (scalaVersion0, scalaBinaryVersion0) = scalaVersion() match {
-      case Milestone213(_, _) => ("2.13.0-M2", "2.13.0-M2")
-      case _ => (scalaVersion(), mill.scalalib.api.Util.scalaBinaryVersion(scalaVersion()))
-    }
-
-    val (bridgeDep, bridgeName, bridgeVersion) =
-      if (isDotty(scalaVersion0)) {
-        val org = scalaOrganization()
-        val name = "dotty-sbt-bridge"
-        val version = scalaVersion()
-        (ivy"$org:$name:$version", name, version)
-      } else {
-        val org = "org.scala-sbt"
-        val name = "compiler-bridge"
-        val version = Versions.zinc
-        (ivy"$org::$name:$version", s"${name}_$scalaBinaryVersion0", version)
-      }
-
-    resolveDependencies(
-      repositories,
-      Lib.depToDependency(_, scalaVersion0, platformSuffix()),
-      Seq(bridgeDep),
-      sources = true
-    ).map(deps =>
-      mill.scalalib.api.Util.grepJar(deps.map(_.path), bridgeName, bridgeVersion, sources = true)
-    )
-  }
 
   /**
     * The local classpath of Scala compiler plugins on-disk; you can add
@@ -159,8 +130,8 @@ trait ScalaModule extends JavaModule { outer =>
       compileClasspath().map(_.path),
       javacOptions(),
       scalaVersion(),
+      scalaOrganization(),
       scalacOptions(),
-      scalaCompilerBridgeSources(),
       scalaCompilerClasspath().map(_.path),
       scalacPluginClasspath().map(_.path),
     )
@@ -187,7 +158,7 @@ trait ScalaModule extends JavaModule { outer =>
     else {
       zincWorker.worker().docJar(
         scalaVersion(),
-        scalaCompilerBridgeSources(),
+        scalaOrganization(),
         scalaCompilerClasspath().map(_.path),
         scalacPluginClasspath().map(_.path),
         files ++ options
diff --git a/scalalib/src/ZincWorkerModule.scala b/scalalib/src/ZincWorkerModule.scala
index 5ca824ce..97d84aaf 100644
--- a/scalalib/src/ZincWorkerModule.scala
+++ b/scalalib/src/ZincWorkerModule.scala
@@ -4,9 +4,12 @@ import coursier.Cache
 import coursier.maven.MavenRepository
 import mill.Agg
 import mill.T
+import mill.api.KeyedLockedCache
 import mill.define.{Discover, Worker}
 import mill.scalalib.Lib.resolveDependencies
-import mill.util.Loose
+import mill.scalalib.api.Util.isDotty
+import mill.scalalib.api.ZincWorkerApi
+import mill.api.Loose
 import mill.util.JsonFormatters._
 
 object ZincWorkerModule extends mill.define.ExternalModule with ZincWorkerModule{
@@ -40,11 +43,61 @@ trait ZincWorkerModule extends mill.Module{
       getClass.getClassLoader
     )
     val cls = cl.loadClass("mill.scalalib.worker.ZincWorkerImpl")
-    val instance = cls.getConstructor(classOf[mill.api.Ctx], classOf[Array[String]])
-      .newInstance(T.ctx(), compilerInterfaceClasspath().map(_.path.toString).toArray[String])
+    val instance = cls.getConstructor(
+      classOf[
+        Either[
+          (ZincWorkerApi.Ctx, Array[os.Path], (String, String) => os.Path),
+          String => os.Path
+        ]
+      ],
+      classOf[(Agg[os.Path], String) => os.Path],
+      classOf[(Agg[os.Path], String) => os.Path],
+      classOf[KeyedLockedCache[_]]
+    )
+      .newInstance(
+        Left((
+          T.ctx(),
+          compilerInterfaceClasspath().map(_.path).toArray,
+          (x: String, y: String) => scalaCompilerBridgeSourceJar(x, y).asSuccess.get.value
+        )),
+        mill.scalalib.api.Util.grepJar(_, "scala-library", _, sources = false),
+        mill.scalalib.api.Util.grepJar(_, "scala-compiler", _, sources = false),
+        new KeyedLockedCache.RandomBoundedCache(1, 1)
+      )
     instance.asInstanceOf[mill.scalalib.api.ZincWorkerApi]
   }
 
+  private val Milestone213 = raw"""2.13.(\d+)-M(\d+)""".r
+  def scalaCompilerBridgeSourceJar(scalaVersion: String,
+                                   scalaOrganization: String) = {
+    val (scalaVersion0, scalaBinaryVersion0) = scalaVersion match {
+      case Milestone213(_, _) => ("2.13.0-M2", "2.13.0-M2")
+      case _ => (scalaVersion, mill.scalalib.api.Util.scalaBinaryVersion(scalaVersion))
+    }
+
+    val (bridgeDep, bridgeName, bridgeVersion) =
+      if (isDotty(scalaVersion0)) {
+        val org = scalaOrganization
+        val name = "dotty-sbt-bridge"
+        val version = scalaVersion
+        (ivy"$org:$name:$version", name, version)
+      } else {
+        val org = "org.scala-sbt"
+        val name = "compiler-bridge"
+        val version = Versions.zinc
+        (ivy"$org::$name:$version", s"${name}_$scalaBinaryVersion0", version)
+      }
+
+    resolveDependencies(
+      repositories,
+      Lib.depToDependency(_, scalaVersion0, ""),
+      Seq(bridgeDep),
+      sources = true
+    ).map(deps =>
+      mill.scalalib.api.Util.grepJar(deps.map(_.path), bridgeName, bridgeVersion, sources = true)
+    )
+  }
+
   def compilerInterfaceClasspath = T{
     resolveDependencies(
       repositories,
diff --git a/scalalib/src/dependency/versions/VersionsFinder.scala b/scalalib/src/dependency/versions/VersionsFinder.scala
index a831ffc3..a9ecc763 100644
--- a/scalalib/src/dependency/versions/VersionsFinder.scala
+++ b/scalalib/src/dependency/versions/VersionsFinder.scala
@@ -5,7 +5,7 @@ import mill.eval.Evaluator
 import mill.scalalib.dependency.metadata.MetadataLoaderFactory
 import mill.scalalib.{Dep, JavaModule, Lib}
 import mill.api.Ctx.{Home, Log}
-import mill.util.{Loose, Strict}
+import mill.api.{Loose, Strict}
 
 private[dependency] object VersionsFinder {
 
diff --git a/scalalib/src/publish/Ivy.scala b/scalalib/src/publish/Ivy.scala
index 22e26ff6..e06efadd 100644
--- a/scalalib/src/publish/Ivy.scala
+++ b/scalalib/src/publish/Ivy.scala
@@ -1,6 +1,6 @@
 package mill.scalalib.publish
 
-import mill.util.Loose.Agg
+import mill.api.Loose.Agg
 
 import scala.xml.PrettyPrinter
 
diff --git a/scalalib/src/publish/Pom.scala b/scalalib/src/publish/Pom.scala
index 57a0e196..a7f1f6fc 100644
--- a/scalalib/src/publish/Pom.scala
+++ b/scalalib/src/publish/Pom.scala
@@ -1,6 +1,6 @@
 package mill.scalalib.publish
 
-import mill.util.Loose.Agg
+import mill.api.Loose.Agg
 
 import scala.xml.{Atom, Elem, NodeSeq, PrettyPrinter}