From de175e69977082e35539097a54d381e465dddf8e Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Tue, 18 Dec 2018 21:11:47 -0800 Subject: Generalize Zinc Worker (#514) * Generalize Zinc worker - Compiler bridges can now be either pre-compiled or on-demand-compiled - Scala library/compiler jar discovery is now configurable - Zinc compiler cache is now configurable, rather than being hardcoded at n=1 * . * update constructor args * remove duplicate util/AggWrapper.scala file * fix * fix * fix * cleanup --- main/api/src/mill/api/KeyedLockedCache.scala | 44 ++++++++++ main/core/src/define/Graph.scala | 2 +- main/core/src/eval/Evaluator.scala | 2 +- main/core/src/util/AggWrapper.scala | 119 --------------------------- main/core/src/util/MultiBiMap.scala | 2 +- main/src/main/ReplApplyHandler.scala | 2 +- main/src/main/RunScript.scala | 2 +- main/src/modules/Jvm.scala | 2 +- main/src/modules/Util.scala | 5 +- main/src/package.scala | 4 +- main/test/resources/examples/javac/build.sc | 2 +- main/test/src/define/CacherTests.scala | 2 +- main/test/src/define/GraphTests.scala | 2 +- main/test/src/eval/EvaluationTests.scala | 2 +- main/test/src/eval/JavaCompileJarTests.scala | 7 +- main/test/src/util/TestEvaluator.scala | 2 +- main/test/src/util/TestUtil.scala | 2 +- 17 files changed, 65 insertions(+), 138 deletions(-) create mode 100644 main/api/src/mill/api/KeyedLockedCache.scala delete mode 100644 main/core/src/util/AggWrapper.scala (limited to 'main') diff --git a/main/api/src/mill/api/KeyedLockedCache.scala b/main/api/src/mill/api/KeyedLockedCache.scala new file mode 100644 index 00000000..47fdd888 --- /dev/null +++ b/main/api/src/mill/api/KeyedLockedCache.scala @@ -0,0 +1,44 @@ +package mill.api + +/** + * A combination lock & cache; users provide a key, value-factory, and a + * body function to be called with the value. [[KeyedLockedCache]] ensures that + * the body function is called with the computed/cached value sequentially. + */ +trait KeyedLockedCache[T]{ + def withCachedValue[V](key: Long)(f: => T)(f2: T => V): V +} + +object KeyedLockedCache{ + class RandomBoundedCache[T](hotParallelism: Int, coldCacheSize: Int) extends KeyedLockedCache[T]{ + private[this] val random = new scala.util.Random(313373) + val available = new java.util.concurrent.Semaphore(hotParallelism) + + // Awful asymptotic complexity, but our caches are tiny n < 10 so it doesn't matter + var cache = Array.fill[Option[(Long, T)]](coldCacheSize)(None) + + def withCachedValue[V](key: Long)(f: => T)(f2: T => V): V = { + available.acquire() + val pickedValue = synchronized{ + cache.indexWhere(_.exists(_._1 == key)) match { + case -1 => f + case i => + val (k, v) = cache(i).get + cache(i) = None + v + } + } + val result = f2(pickedValue) + synchronized{ + cache.indexWhere(_.isEmpty) match{ + // Random eviction #YOLO + case -1 => cache(random.nextInt(cache.length)) = Some((key, pickedValue)) + case i => cache(i) = Some((key, pickedValue)) + } + } + + available.release() + result + } + } +} diff --git a/main/core/src/define/Graph.scala b/main/core/src/define/Graph.scala index 3119f2fb..5b29bd7b 100644 --- a/main/core/src/define/Graph.scala +++ b/main/core/src/define/Graph.scala @@ -2,7 +2,7 @@ package mill.define import mill.eval.Tarjans import mill.util.MultiBiMap -import mill.util.Strict.Agg +import mill.api.Strict.Agg object Graph { diff --git a/main/core/src/eval/Evaluator.scala b/main/core/src/eval/Evaluator.scala index 8709064e..dbaf9433 100644 --- a/main/core/src/eval/Evaluator.scala +++ b/main/core/src/eval/Evaluator.scala @@ -10,7 +10,7 @@ import mill.define.{Ctx => _, _} import mill.api.Result.OuterStack import mill.util import mill.util._ -import mill.util.Strict.Agg +import mill.api.Strict.Agg import scala.collection.mutable import scala.util.control.NonFatal diff --git a/main/core/src/util/AggWrapper.scala b/main/core/src/util/AggWrapper.scala deleted file mode 100644 index 6c107875..00000000 --- a/main/core/src/util/AggWrapper.scala +++ /dev/null @@ -1,119 +0,0 @@ -package mill.util - - - -import scala.collection.mutable -object Strict extends AggWrapper(true) -object Loose extends AggWrapper(false) -sealed class AggWrapper(strictUniqueness: Boolean){ - /** - * A collection with enforced uniqueness, fast contains and deterministic - * ordering. Raises an exception if a duplicate is found; call - * `toSeq.distinct` if you explicitly want to make it swallow duplicates - */ - trait Agg[V] extends TraversableOnce[V]{ - def contains(v: V): Boolean - def items: Iterator[V] - def indexed: IndexedSeq[V] - def flatMap[T](f: V => TraversableOnce[T]): Agg[T] - def map[T](f: V => T): Agg[T] - def filter(f: V => Boolean): Agg[V] - def withFilter(f: V => Boolean): Agg[V] - def collect[T](f: PartialFunction[V, T]): Agg[T] - def zipWithIndex: Agg[(V, Int)] - def reverse: Agg[V] - def zip[T](other: Agg[T]): Agg[(V, T)] - def ++[T >: V](other: TraversableOnce[T]): Agg[T] - def length: Int - } - - object Agg{ - def empty[V]: Agg[V] = new Agg.Mutable[V] - implicit def jsonFormat[T: upickle.default.ReadWriter]: upickle.default.ReadWriter[Agg[T]] = - upickle.default.readwriter[Seq[T]].bimap[Agg[T]]( - _.toList, - Agg.from(_) - ) - - def apply[V](items: V*) = from(items) - - implicit def from[V](items: TraversableOnce[V]): Agg[V] = { - val set = new Agg.Mutable[V]() - items.foreach(set.append) - set - } - - - class Mutable[V]() extends Agg[V]{ - - private[this] val set0 = mutable.LinkedHashSet.empty[V] - def contains(v: V) = set0.contains(v) - def append(v: V) = if (!contains(v)){ - set0.add(v) - - }else if (strictUniqueness){ - throw new Exception("Duplicated item inserted into OrderedSet: " + v) - } - def appendAll(vs: Seq[V]) = vs.foreach(append) - def items = set0.iterator - def indexed: IndexedSeq[V] = items.toIndexedSeq - def set: collection.Set[V] = set0 - - def map[T](f: V => T): Agg[T] = { - val output = new Agg.Mutable[T] - for(i <- items) output.append(f(i)) - output - } - def flatMap[T](f: V => TraversableOnce[T]): Agg[T] = { - val output = new Agg.Mutable[T] - for(i <- items) for(i0 <- f(i)) output.append(i0) - output - } - def filter(f: V => Boolean): Agg[V] = { - val output = new Agg.Mutable[V] - for(i <- items) if (f(i)) output.append(i) - output - } - def withFilter(f: V => Boolean): Agg[V] = filter(f) - - def collect[T](f: PartialFunction[V, T]) = this.filter(f.isDefinedAt).map(x => f(x)) - - def zipWithIndex = { - var i = 0 - this.map{ x => - i += 1 - (x, i-1) - } - } - - def reverse = Agg.from(indexed.reverseIterator) - - def zip[T](other: Agg[T]) = Agg.from(items.zip(other.items)) - def ++[T >: V](other: TraversableOnce[T]) = Agg.from(items ++ other) - def length: Int = set0.size - - // Members declared in scala.collection.GenTraversableOnce - def isTraversableAgain: Boolean = items.isTraversableAgain - def toIterator: Iterator[V] = items.toIterator - def toStream: Stream[V] = items.toStream - - // Members declared in scala.collection.TraversableOnce - def copyToArray[B >: V](xs: Array[B], start: Int,len: Int): Unit = items.copyToArray(xs, start, len) - def exists(p: V => Boolean): Boolean = items.exists(p) - def find(p: V => Boolean): Option[V] = items.find(p) - def forall(p: V => Boolean): Boolean = items.forall(p) - def foreach[U](f: V => U): Unit = items.foreach(f) - def hasDefiniteSize: Boolean = items.hasDefiniteSize - def isEmpty: Boolean = items.isEmpty - def seq: scala.collection.TraversableOnce[V] = items - def toTraversable: Traversable[V] = items.toTraversable - - override def hashCode() = items.map(_.hashCode()).sum - override def equals(other: Any) = other match{ - case s: Agg[_] => items.sameElements(s.items) - case _ => super.equals(other) - } - override def toString = items.mkString("Agg(", ", ", ")") - } - } -} diff --git a/main/core/src/util/MultiBiMap.scala b/main/core/src/util/MultiBiMap.scala index 73bb42c4..51ea63f2 100644 --- a/main/core/src/util/MultiBiMap.scala +++ b/main/core/src/util/MultiBiMap.scala @@ -1,7 +1,7 @@ package mill.util import scala.collection.mutable -import Strict.Agg +import mill.api.Strict.Agg /** * A map from keys to collections of values: you can assign multiple values diff --git a/main/src/main/ReplApplyHandler.scala b/main/src/main/ReplApplyHandler.scala index a8e467d4..786a1409 100644 --- a/main/src/main/ReplApplyHandler.scala +++ b/main/src/main/ReplApplyHandler.scala @@ -6,7 +6,7 @@ import mill.define.Segment.Label import mill.define._ import mill.eval.{Evaluator, Result} -import mill.util.Strict.Agg +import mill.api.Strict.Agg import scala.collection.mutable object ReplApplyHandler{ diff --git a/main/src/main/RunScript.scala b/main/src/main/RunScript.scala index 47526631..b858c8b9 100644 --- a/main/src/main/RunScript.scala +++ b/main/src/main/RunScript.scala @@ -11,7 +11,7 @@ import mill.define._ import mill.eval.{Evaluator, PathRef, Result} import mill.util.{EitherOps, ParseArgs, Watched} import mill.api.Logger -import mill.util.Strict.Agg +import mill.api.Strict.Agg import scala.collection.mutable import scala.reflect.ClassTag diff --git a/main/src/modules/Jvm.scala b/main/src/modules/Jvm.scala index 1a51ed8b..e17631e3 100644 --- a/main/src/modules/Jvm.scala +++ b/main/src/modules/Jvm.scala @@ -15,7 +15,7 @@ import mill.main.client.InputPumper import mill.eval.{PathRef, Result} import mill.util.Ctx import mill.api.IO -import mill.util.Loose.Agg +import mill.api.Loose.Agg import scala.collection.mutable import scala.collection.JavaConverters._ diff --git a/main/src/modules/Util.scala b/main/src/modules/Util.scala index 2b98a304..029626fe 100644 --- a/main/src/modules/Util.scala +++ b/main/src/modules/Util.scala @@ -3,7 +3,8 @@ package mill.modules import coursier.Repository import mill.api.{PathRef, IO} -import mill.util.{Ctx, Loose} +import mill.util.Ctx +import mill.api.Loose object Util { @@ -55,7 +56,7 @@ object Util { val localPath = sys.props(key) if (localPath != null) { mill.api.Result.Success( - Loose.Agg.from(localPath.split(',').map(p => PathRef(os.Path(p), quick = true))) + mill.api.Loose.Agg.from(localPath.split(',').map(p => PathRef(os.Path(p), quick = true))) ) } else { mill.modules.Jvm.resolveDependencies( diff --git a/main/src/package.scala b/main/src/package.scala index 0ccd094f..6bcb1bdf 100644 --- a/main/src/package.scala +++ b/main/src/package.scala @@ -7,6 +7,6 @@ package object mill extends JsonFormatters{ type PathRef = mill.api.PathRef type Module = define.Module type Cross[T] = define.Cross[T] - type Agg[T] = util.Loose.Agg[T] - val Agg = util.Loose.Agg + type Agg[T] = mill.api.Loose.Agg[T] + val Agg = mill.api.Loose.Agg } diff --git a/main/test/resources/examples/javac/build.sc b/main/test/resources/examples/javac/build.sc index 2ed9f915..17366219 100644 --- a/main/test/resources/examples/javac/build.sc +++ b/main/test/resources/examples/javac/build.sc @@ -2,7 +2,7 @@ import mill.T import mill.eval.JavaCompileJarTests.compileAll import mill.api.PathRef import mill.modules.Jvm -import mill.util.Loose +import mill.api.Loose def sourceRootPath = millSourcePath / 'src def resourceRootPath = millSourcePath / 'resources diff --git a/main/test/src/define/CacherTests.scala b/main/test/src/define/CacherTests.scala index 59ebf3f6..1524e5c1 100644 --- a/main/test/src/define/CacherTests.scala +++ b/main/test/src/define/CacherTests.scala @@ -1,7 +1,7 @@ package mill.define import mill.util.{DummyLogger, TestEvaluator, TestUtil} -import mill.util.Strict.Agg +import mill.api.Strict.Agg import mill.T import mill.api.Result.Success import utest._ diff --git a/main/test/src/define/GraphTests.scala b/main/test/src/define/GraphTests.scala index 224ce59f..b36dbf95 100644 --- a/main/test/src/define/GraphTests.scala +++ b/main/test/src/define/GraphTests.scala @@ -4,7 +4,7 @@ package mill.define import mill.eval.Evaluator import mill.util.{TestGraphs, TestUtil} import utest._ -import mill.util.Strict.Agg +import mill.api.Strict.Agg object GraphTests extends TestSuite{ val tests = Tests{ diff --git a/main/test/src/eval/EvaluationTests.scala b/main/test/src/eval/EvaluationTests.scala index 74f9088c..7f924db2 100644 --- a/main/test/src/eval/EvaluationTests.scala +++ b/main/test/src/eval/EvaluationTests.scala @@ -5,7 +5,7 @@ import mill.util.TestUtil.{Test, test} import mill.define.{Discover, Graph, Target, Task} import mill.{Module, T} import mill.util.{DummyLogger, TestEvaluator, TestGraphs, TestUtil} -import mill.util.Strict.Agg +import mill.api.Strict.Agg import utest._ import utest.framework.TestPath diff --git a/main/test/src/eval/JavaCompileJarTests.scala b/main/test/src/eval/JavaCompileJarTests.scala index 426c6ea6..0f9002df 100644 --- a/main/test/src/eval/JavaCompileJarTests.scala +++ b/main/test/src/eval/JavaCompileJarTests.scala @@ -4,12 +4,13 @@ import mill.define.{Discover, Input, Target, Task} import mill.modules.Jvm import mill.api.Ctx.Dest import mill.{Module, T} -import mill.util.{DummyLogger, Loose, TestEvaluator, TestUtil} -import mill.util.Strict.Agg +import mill.util.{DummyLogger, TestEvaluator, TestUtil} +import mill.api.Strict.Agg +import mill.api.Loose import utest._ import mill._ object JavaCompileJarTests extends TestSuite{ - def compileAll(sources: mill.util.Loose.Agg[PathRef])(implicit ctx: Dest) = { + def compileAll(sources: mill.api.Loose.Agg[PathRef])(implicit ctx: Dest) = { os.makeDir.all(ctx.dest) os.proc("javac", sources.map(_.path.toString()).toSeq, "-d", ctx.dest).call(ctx.dest) diff --git a/main/test/src/util/TestEvaluator.scala b/main/test/src/util/TestEvaluator.scala index 9a235679..81c8fe12 100644 --- a/main/test/src/util/TestEvaluator.scala +++ b/main/test/src/util/TestEvaluator.scala @@ -3,7 +3,7 @@ package mill.util import mill.define.{Input, Target, Task} import mill.api.Result.OuterStack import mill.eval.{Evaluator, Result} -import mill.util.Strict.Agg +import mill.api.Strict.Agg import utest.assert import utest.framework.TestPath diff --git a/main/test/src/util/TestUtil.scala b/main/test/src/util/TestUtil.scala index baab2992..462b7f5c 100644 --- a/main/test/src/util/TestUtil.scala +++ b/main/test/src/util/TestUtil.scala @@ -5,7 +5,7 @@ import mill.define._ import mill.api.Result import mill.api.Result.OuterStack import utest.assert -import mill.util.Strict.Agg +import mill.api.Strict.Agg import utest.framework.TestPath import scala.collection.mutable -- cgit v1.2.3