Generalize Zinc Worker (#514)

* Generalize Zinc worker

- Compiler bridges can now be either pre-compiled or on-demand-compiled
- Scala library/compiler jar discovery is now configurable
- Zinc compiler cache is now configurable, rather than being hardcoded at n=1

* .

* update constructor args

* remove duplicate util/AggWrapper.scala file

* fix

* fix

* fix

* cleanup

17 files changed, 65 insertions, 138 deletions

diff --git a/main/api/src/mill/api/KeyedLockedCache.scala b/main/api/src/mill/api/KeyedLockedCache.scala
new file mode 100644
index 00000000..47fdd888
--- /dev/null
+++ b/main/api/src/mill/api/KeyedLockedCache.scala
@@ -0,0 +1,44 @@
+package mill.api
+
+/**
+  * A combination lock & cache; users provide a key, value-factory, and a
+  * body function to be called with the value. [[KeyedLockedCache]] ensures that
+  * the body function is called with the computed/cached value sequentially.
+  */
+trait KeyedLockedCache[T]{
+  def withCachedValue[V](key: Long)(f: => T)(f2: T => V): V
+}
+
+object KeyedLockedCache{
+  class RandomBoundedCache[T](hotParallelism: Int, coldCacheSize: Int) extends KeyedLockedCache[T]{
+    private[this] val random = new scala.util.Random(313373)
+    val available = new java.util.concurrent.Semaphore(hotParallelism)
+
+    // Awful asymptotic complexity, but our caches are tiny n < 10 so it doesn't matter
+    var cache = Array.fill[Option[(Long, T)]](coldCacheSize)(None)
+
+    def withCachedValue[V](key: Long)(f: => T)(f2: T => V): V = {
+      available.acquire()
+      val pickedValue = synchronized{
+        cache.indexWhere(_.exists(_._1 == key)) match {
+          case -1 => f
+          case i =>
+            val (k, v) = cache(i).get
+            cache(i) = None
+            v
+        }
+      }
+      val result = f2(pickedValue)
+      synchronized{
+        cache.indexWhere(_.isEmpty) match{
+          // Random eviction #YOLO
+          case -1 => cache(random.nextInt(cache.length)) = Some((key, pickedValue))
+          case i => cache(i) = Some((key, pickedValue))
+        }
+      }
+
+      available.release()
+      result
+    }
+  }
+}
diff --git a/main/core/src/define/Graph.scala b/main/core/src/define/Graph.scala
index 3119f2fb..5b29bd7b 100644
--- a/main/core/src/define/Graph.scala
+++ b/main/core/src/define/Graph.scala
@@ -2,7 +2,7 @@ package mill.define
 
 import mill.eval.Tarjans
 import mill.util.MultiBiMap
-import mill.util.Strict.Agg
+import mill.api.Strict.Agg
 
 object Graph {
 
diff --git a/main/core/src/eval/Evaluator.scala b/main/core/src/eval/Evaluator.scala
index 8709064e..dbaf9433 100644
--- a/main/core/src/eval/Evaluator.scala
+++ b/main/core/src/eval/Evaluator.scala
@@ -10,7 +10,7 @@ import mill.define.{Ctx => _, _}
 import mill.api.Result.OuterStack
 import mill.util
 import mill.util._
-import mill.util.Strict.Agg
+import mill.api.Strict.Agg
 
 import scala.collection.mutable
 import scala.util.control.NonFatal
diff --git a/main/core/src/util/AggWrapper.scala b/main/core/src/util/AggWrapper.scala
deleted file mode 100644
index 6c107875..00000000
--- a/main/core/src/util/AggWrapper.scala
+++ /dev/null
@@ -1,119 +0,0 @@
-package mill.util
-
-
-
-import scala.collection.mutable
-object Strict extends AggWrapper(true)
-object Loose extends AggWrapper(false)
-sealed class AggWrapper(strictUniqueness: Boolean){
-  /**
-    * A collection with enforced uniqueness, fast contains and deterministic
-    * ordering. Raises an exception if a duplicate is found; call
-    * `toSeq.distinct` if you explicitly want to make it swallow duplicates
-    */
-  trait Agg[V] extends TraversableOnce[V]{
-    def contains(v: V): Boolean
-    def items: Iterator[V]
-    def indexed: IndexedSeq[V]
-    def flatMap[T](f: V => TraversableOnce[T]): Agg[T]
-    def map[T](f: V => T): Agg[T]
-    def filter(f: V => Boolean): Agg[V]
-    def withFilter(f: V => Boolean): Agg[V]
-    def collect[T](f: PartialFunction[V, T]): Agg[T]
-    def zipWithIndex: Agg[(V, Int)]
-    def reverse: Agg[V]
-    def zip[T](other: Agg[T]): Agg[(V, T)]
-    def ++[T >: V](other: TraversableOnce[T]): Agg[T]
-    def length: Int
-  }
-
-  object Agg{
-    def empty[V]: Agg[V] = new Agg.Mutable[V]
-    implicit def jsonFormat[T: upickle.default.ReadWriter]: upickle.default.ReadWriter[Agg[T]] =
-      upickle.default.readwriter[Seq[T]].bimap[Agg[T]](
-        _.toList,
-        Agg.from(_)
-      )
-
-    def apply[V](items: V*) = from(items)
-
-    implicit def from[V](items: TraversableOnce[V]): Agg[V] = {
-      val set = new Agg.Mutable[V]()
-      items.foreach(set.append)
-      set
-    }
-
-
-    class Mutable[V]() extends Agg[V]{
-
-      private[this] val set0 = mutable.LinkedHashSet.empty[V]
-      def contains(v: V) = set0.contains(v)
-      def append(v: V) = if (!contains(v)){
-        set0.add(v)
-
-      }else if (strictUniqueness){
-        throw new Exception("Duplicated item inserted into OrderedSet: " + v)
-      }
-      def appendAll(vs: Seq[V]) = vs.foreach(append)
-      def items = set0.iterator
-      def indexed: IndexedSeq[V] = items.toIndexedSeq
-      def set: collection.Set[V] = set0
-
-      def map[T](f: V => T): Agg[T] = {
-        val output = new Agg.Mutable[T]
-        for(i <- items) output.append(f(i))
-        output
-      }
-      def flatMap[T](f: V => TraversableOnce[T]): Agg[T] = {
-        val output = new Agg.Mutable[T]
-        for(i <- items) for(i0 <- f(i)) output.append(i0)
-        output
-      }
-      def filter(f: V => Boolean): Agg[V] = {
-        val output = new Agg.Mutable[V]
-        for(i <- items) if (f(i)) output.append(i)
-        output
-      }
-      def withFilter(f: V => Boolean): Agg[V] = filter(f)
-
-      def collect[T](f: PartialFunction[V, T]) = this.filter(f.isDefinedAt).map(x => f(x))
-
-      def zipWithIndex = {
-        var i = 0
-        this.map{ x =>
-          i += 1
-          (x, i-1)
-        }
-      }
-
-      def reverse = Agg.from(indexed.reverseIterator)
-
-      def zip[T](other: Agg[T]) = Agg.from(items.zip(other.items))
-      def ++[T >: V](other: TraversableOnce[T]) = Agg.from(items ++ other)
-      def length: Int = set0.size
-
-      // Members declared in scala.collection.GenTraversableOnce
-      def isTraversableAgain: Boolean = items.isTraversableAgain
-      def toIterator: Iterator[V] = items.toIterator
-      def toStream: Stream[V] = items.toStream
-
-      // Members declared in scala.collection.TraversableOnce
-      def copyToArray[B >: V](xs: Array[B], start: Int,len: Int): Unit = items.copyToArray(xs, start, len)
-      def exists(p: V => Boolean): Boolean = items.exists(p)
-      def find(p: V => Boolean): Option[V] = items.find(p)
-      def forall(p: V => Boolean): Boolean = items.forall(p)
-      def foreach[U](f: V => U): Unit = items.foreach(f)
-      def hasDefiniteSize: Boolean = items.hasDefiniteSize
-      def isEmpty: Boolean = items.isEmpty
-      def seq: scala.collection.TraversableOnce[V] = items
-      def toTraversable: Traversable[V] = items.toTraversable
-
-      override def hashCode() = items.map(_.hashCode()).sum
-      override def equals(other: Any) = other match{
-        case s: Agg[_] => items.sameElements(s.items)
-        case _ => super.equals(other)
-      }
-      override def toString = items.mkString("Agg(", ", ", ")")
-    }
-  }
-}
diff --git a/main/core/src/util/MultiBiMap.scala b/main/core/src/util/MultiBiMap.scala
index 73bb42c4..51ea63f2 100644
--- a/main/core/src/util/MultiBiMap.scala
+++ b/main/core/src/util/MultiBiMap.scala
@@ -1,7 +1,7 @@
 package mill.util
 
 import scala.collection.mutable
-import Strict.Agg
+import mill.api.Strict.Agg
 
 /**
   * A map from keys to collections of values: you can assign multiple values
diff --git a/main/src/main/ReplApplyHandler.scala b/main/src/main/ReplApplyHandler.scala
index a8e467d4..786a1409 100644
--- a/main/src/main/ReplApplyHandler.scala
+++ b/main/src/main/ReplApplyHandler.scala
@@ -6,7 +6,7 @@ import mill.define.Segment.Label
 import mill.define._
 import mill.eval.{Evaluator, Result}
 
-import mill.util.Strict.Agg
+import mill.api.Strict.Agg
 
 import scala.collection.mutable
 object ReplApplyHandler{
diff --git a/main/src/main/RunScript.scala b/main/src/main/RunScript.scala
index 47526631..b858c8b9 100644
--- a/main/src/main/RunScript.scala
+++ b/main/src/main/RunScript.scala
@@ -11,7 +11,7 @@ import mill.define._
 import mill.eval.{Evaluator, PathRef, Result}
 import mill.util.{EitherOps, ParseArgs, Watched}
 import mill.api.Logger
-import mill.util.Strict.Agg
+import mill.api.Strict.Agg
 
 import scala.collection.mutable
 import scala.reflect.ClassTag
diff --git a/main/src/modules/Jvm.scala b/main/src/modules/Jvm.scala
index 1a51ed8b..e17631e3 100644
--- a/main/src/modules/Jvm.scala
+++ b/main/src/modules/Jvm.scala
@@ -15,7 +15,7 @@ import mill.main.client.InputPumper
 import mill.eval.{PathRef, Result}
 import mill.util.Ctx
 import mill.api.IO
-import mill.util.Loose.Agg
+import mill.api.Loose.Agg
 
 import scala.collection.mutable
 import scala.collection.JavaConverters._
diff --git a/main/src/modules/Util.scala b/main/src/modules/Util.scala
index 2b98a304..029626fe 100644
--- a/main/src/modules/Util.scala
+++ b/main/src/modules/Util.scala
@@ -3,7 +3,8 @@ package mill.modules
 
 import coursier.Repository
 import mill.api.{PathRef, IO}
-import mill.util.{Ctx, Loose}
+import mill.util.Ctx
+import mill.api.Loose
 
 
 object Util {
@@ -55,7 +56,7 @@ object Util {
     val localPath = sys.props(key)
     if (localPath != null) {
       mill.api.Result.Success(
-        Loose.Agg.from(localPath.split(',').map(p => PathRef(os.Path(p), quick = true)))
+        mill.api.Loose.Agg.from(localPath.split(',').map(p => PathRef(os.Path(p), quick = true)))
       )
     } else {
       mill.modules.Jvm.resolveDependencies(
diff --git a/main/src/package.scala b/main/src/package.scala
index 0ccd094f..6bcb1bdf 100644
--- a/main/src/package.scala
+++ b/main/src/package.scala
@@ -7,6 +7,6 @@ package object mill extends JsonFormatters{
   type PathRef = mill.api.PathRef
   type Module = define.Module
   type Cross[T] = define.Cross[T]
-  type Agg[T] = util.Loose.Agg[T]
-  val Agg = util.Loose.Agg
+  type Agg[T] = mill.api.Loose.Agg[T]
+  val Agg = mill.api.Loose.Agg
 }
diff --git a/main/test/resources/examples/javac/build.sc b/main/test/resources/examples/javac/build.sc
index 2ed9f915..17366219 100644
--- a/main/test/resources/examples/javac/build.sc
+++ b/main/test/resources/examples/javac/build.sc
@@ -2,7 +2,7 @@ import mill.T
 import mill.eval.JavaCompileJarTests.compileAll
 import mill.api.PathRef
 import mill.modules.Jvm
-import mill.util.Loose
+import mill.api.Loose
 
 def sourceRootPath = millSourcePath / 'src
 def resourceRootPath = millSourcePath / 'resources
diff --git a/main/test/src/define/CacherTests.scala b/main/test/src/define/CacherTests.scala
index 59ebf3f6..1524e5c1 100644
--- a/main/test/src/define/CacherTests.scala
+++ b/main/test/src/define/CacherTests.scala
@@ -1,7 +1,7 @@
 package mill.define
 
 import mill.util.{DummyLogger, TestEvaluator, TestUtil}
-import mill.util.Strict.Agg
+import mill.api.Strict.Agg
 import mill.T
 import mill.api.Result.Success
 import utest._
diff --git a/main/test/src/define/GraphTests.scala b/main/test/src/define/GraphTests.scala
index 224ce59f..b36dbf95 100644
--- a/main/test/src/define/GraphTests.scala
+++ b/main/test/src/define/GraphTests.scala
@@ -4,7 +4,7 @@ package mill.define
 import mill.eval.Evaluator
 import mill.util.{TestGraphs, TestUtil}
 import utest._
-import mill.util.Strict.Agg
+import mill.api.Strict.Agg
 object GraphTests extends TestSuite{
 
   val tests = Tests{
diff --git a/main/test/src/eval/EvaluationTests.scala b/main/test/src/eval/EvaluationTests.scala
index 74f9088c..7f924db2 100644
--- a/main/test/src/eval/EvaluationTests.scala
+++ b/main/test/src/eval/EvaluationTests.scala
@@ -5,7 +5,7 @@ import mill.util.TestUtil.{Test, test}
 import mill.define.{Discover, Graph, Target, Task}
 import mill.{Module, T}
 import mill.util.{DummyLogger, TestEvaluator, TestGraphs, TestUtil}
-import mill.util.Strict.Agg
+import mill.api.Strict.Agg
 import utest._
 import utest.framework.TestPath
 
diff --git a/main/test/src/eval/JavaCompileJarTests.scala b/main/test/src/eval/JavaCompileJarTests.scala
index 426c6ea6..0f9002df 100644
--- a/main/test/src/eval/JavaCompileJarTests.scala
+++ b/main/test/src/eval/JavaCompileJarTests.scala
@@ -4,12 +4,13 @@ import mill.define.{Discover, Input, Target, Task}
 import mill.modules.Jvm
 import mill.api.Ctx.Dest
 import mill.{Module, T}
-import mill.util.{DummyLogger, Loose, TestEvaluator, TestUtil}
-import mill.util.Strict.Agg
+import mill.util.{DummyLogger, TestEvaluator, TestUtil}
+import mill.api.Strict.Agg
+import mill.api.Loose
 import utest._
 import mill._
 object JavaCompileJarTests extends TestSuite{
-  def compileAll(sources: mill.util.Loose.Agg[PathRef])(implicit ctx: Dest) = {
+  def compileAll(sources: mill.api.Loose.Agg[PathRef])(implicit ctx: Dest) = {
     os.makeDir.all(ctx.dest)
 
     os.proc("javac", sources.map(_.path.toString()).toSeq, "-d", ctx.dest).call(ctx.dest)
diff --git a/main/test/src/util/TestEvaluator.scala b/main/test/src/util/TestEvaluator.scala
index 9a235679..81c8fe12 100644
--- a/main/test/src/util/TestEvaluator.scala
+++ b/main/test/src/util/TestEvaluator.scala
@@ -3,7 +3,7 @@ package mill.util
 import mill.define.{Input, Target, Task}
 import mill.api.Result.OuterStack
 import mill.eval.{Evaluator, Result}
-import mill.util.Strict.Agg
+import mill.api.Strict.Agg
 import utest.assert
 import utest.framework.TestPath
 
diff --git a/main/test/src/util/TestUtil.scala b/main/test/src/util/TestUtil.scala
index baab2992..462b7f5c 100644
--- a/main/test/src/util/TestUtil.scala
+++ b/main/test/src/util/TestUtil.scala
@@ -5,7 +5,7 @@ import mill.define._
 import mill.api.Result
 import mill.api.Result.OuterStack
 import utest.assert
-import mill.util.Strict.Agg
+import mill.api.Strict.Agg
 import utest.framework.TestPath
 
 import scala.collection.mutable