diff options
33 files changed, 248 insertions, 291 deletions
diff --git a/ci/shared.sc b/ci/shared.sc index 2f133486..a496fd1f 100644 --- a/ci/shared.sc +++ b/ci/shared.sc @@ -74,7 +74,7 @@ def generateEval(dir: Path) = { s"""package mill.main |import mill.eval.Evaluator |import mill.define.Task - |import mill.util.Strict.Agg + |import mill.api.Strict.Agg |class EvalGenerated(evaluator: Evaluator) { | type TT[+X] = Task[X] | ${(1 to 22).map(generate).mkString("\n")} diff --git a/contrib/scalapblib/src/ScalaPBModule.scala b/contrib/scalapblib/src/ScalaPBModule.scala index db5c5c8b..57bfdd40 100644 --- a/contrib/scalapblib/src/ScalaPBModule.scala +++ b/contrib/scalapblib/src/ScalaPBModule.scala @@ -7,7 +7,7 @@ import mill.define.Sources import mill.api.PathRef import mill.scalalib.Lib.resolveDependencies import mill.scalalib._ -import mill.util.Loose +import mill.api.Loose trait ScalaPBModule extends ScalaModule { diff --git a/contrib/twirllib/src/TwirlModule.scala b/contrib/twirllib/src/TwirlModule.scala index 328afc47..985765fc 100644 --- a/contrib/twirllib/src/TwirlModule.scala +++ b/contrib/twirllib/src/TwirlModule.scala @@ -6,7 +6,7 @@ import mill.define.Sources import mill.api.PathRef import mill.scalalib.Lib.resolveDependencies import mill.scalalib._ -import mill.util.Loose +import mill.api.Loose import scala.io.Codec import scala.util.Properties diff --git a/main/api/src/mill/api/KeyedLockedCache.scala b/main/api/src/mill/api/KeyedLockedCache.scala new file mode 100644 index 00000000..47fdd888 --- /dev/null +++ b/main/api/src/mill/api/KeyedLockedCache.scala @@ -0,0 +1,44 @@ +package mill.api + +/** + * A combination lock & cache; users provide a key, value-factory, and a + * body function to be called with the value. [[KeyedLockedCache]] ensures that + * the body function is called with the computed/cached value sequentially. + */ +trait KeyedLockedCache[T]{ + def withCachedValue[V](key: Long)(f: => T)(f2: T => V): V +} + +object KeyedLockedCache{ + class RandomBoundedCache[T](hotParallelism: Int, coldCacheSize: Int) extends KeyedLockedCache[T]{ + private[this] val random = new scala.util.Random(313373) + val available = new java.util.concurrent.Semaphore(hotParallelism) + + // Awful asymptotic complexity, but our caches are tiny n < 10 so it doesn't matter + var cache = Array.fill[Option[(Long, T)]](coldCacheSize)(None) + + def withCachedValue[V](key: Long)(f: => T)(f2: T => V): V = { + available.acquire() + val pickedValue = synchronized{ + cache.indexWhere(_.exists(_._1 == key)) match { + case -1 => f + case i => + val (k, v) = cache(i).get + cache(i) = None + v + } + } + val result = f2(pickedValue) + synchronized{ + cache.indexWhere(_.isEmpty) match{ + // Random eviction #YOLO + case -1 => cache(random.nextInt(cache.length)) = Some((key, pickedValue)) + case i => cache(i) = Some((key, pickedValue)) + } + } + + available.release() + result + } + } +} diff --git a/main/core/src/define/Graph.scala b/main/core/src/define/Graph.scala index 3119f2fb..5b29bd7b 100644 --- a/main/core/src/define/Graph.scala +++ b/main/core/src/define/Graph.scala @@ -2,7 +2,7 @@ package mill.define import mill.eval.Tarjans import mill.util.MultiBiMap -import mill.util.Strict.Agg +import mill.api.Strict.Agg object Graph { diff --git a/main/core/src/eval/Evaluator.scala b/main/core/src/eval/Evaluator.scala index 8709064e..dbaf9433 100644 --- a/main/core/src/eval/Evaluator.scala +++ b/main/core/src/eval/Evaluator.scala @@ -10,7 +10,7 @@ import mill.define.{Ctx => _, _} import mill.api.Result.OuterStack import mill.util import mill.util._ -import mill.util.Strict.Agg +import mill.api.Strict.Agg import scala.collection.mutable import scala.util.control.NonFatal diff --git a/main/core/src/util/AggWrapper.scala b/main/core/src/util/AggWrapper.scala deleted file mode 100644 index 6c107875..00000000 --- a/main/core/src/util/AggWrapper.scala +++ /dev/null @@ -1,119 +0,0 @@ -package mill.util - - - -import scala.collection.mutable -object Strict extends AggWrapper(true) -object Loose extends AggWrapper(false) -sealed class AggWrapper(strictUniqueness: Boolean){ - /** - * A collection with enforced uniqueness, fast contains and deterministic - * ordering. Raises an exception if a duplicate is found; call - * `toSeq.distinct` if you explicitly want to make it swallow duplicates - */ - trait Agg[V] extends TraversableOnce[V]{ - def contains(v: V): Boolean - def items: Iterator[V] - def indexed: IndexedSeq[V] - def flatMap[T](f: V => TraversableOnce[T]): Agg[T] - def map[T](f: V => T): Agg[T] - def filter(f: V => Boolean): Agg[V] - def withFilter(f: V => Boolean): Agg[V] - def collect[T](f: PartialFunction[V, T]): Agg[T] - def zipWithIndex: Agg[(V, Int)] - def reverse: Agg[V] - def zip[T](other: Agg[T]): Agg[(V, T)] - def ++[T >: V](other: TraversableOnce[T]): Agg[T] - def length: Int - } - - object Agg{ - def empty[V]: Agg[V] = new Agg.Mutable[V] - implicit def jsonFormat[T: upickle.default.ReadWriter]: upickle.default.ReadWriter[Agg[T]] = - upickle.default.readwriter[Seq[T]].bimap[Agg[T]]( - _.toList, - Agg.from(_) - ) - - def apply[V](items: V*) = from(items) - - implicit def from[V](items: TraversableOnce[V]): Agg[V] = { - val set = new Agg.Mutable[V]() - items.foreach(set.append) - set - } - - - class Mutable[V]() extends Agg[V]{ - - private[this] val set0 = mutable.LinkedHashSet.empty[V] - def contains(v: V) = set0.contains(v) - def append(v: V) = if (!contains(v)){ - set0.add(v) - - }else if (strictUniqueness){ - throw new Exception("Duplicated item inserted into OrderedSet: " + v) - } - def appendAll(vs: Seq[V]) = vs.foreach(append) - def items = set0.iterator - def indexed: IndexedSeq[V] = items.toIndexedSeq - def set: collection.Set[V] = set0 - - def map[T](f: V => T): Agg[T] = { - val output = new Agg.Mutable[T] - for(i <- items) output.append(f(i)) - output - } - def flatMap[T](f: V => TraversableOnce[T]): Agg[T] = { - val output = new Agg.Mutable[T] - for(i <- items) for(i0 <- f(i)) output.append(i0) - output - } - def filter(f: V => Boolean): Agg[V] = { - val output = new Agg.Mutable[V] - for(i <- items) if (f(i)) output.append(i) - output - } - def withFilter(f: V => Boolean): Agg[V] = filter(f) - - def collect[T](f: PartialFunction[V, T]) = this.filter(f.isDefinedAt).map(x => f(x)) - - def zipWithIndex = { - var i = 0 - this.map{ x => - i += 1 - (x, i-1) - } - } - - def reverse = Agg.from(indexed.reverseIterator) - - def zip[T](other: Agg[T]) = Agg.from(items.zip(other.items)) - def ++[T >: V](other: TraversableOnce[T]) = Agg.from(items ++ other) - def length: Int = set0.size - - // Members declared in scala.collection.GenTraversableOnce - def isTraversableAgain: Boolean = items.isTraversableAgain - def toIterator: Iterator[V] = items.toIterator - def toStream: Stream[V] = items.toStream - - // Members declared in scala.collection.TraversableOnce - def copyToArray[B >: V](xs: Array[B], start: Int,len: Int): Unit = items.copyToArray(xs, start, len) - def exists(p: V => Boolean): Boolean = items.exists(p) - def find(p: V => Boolean): Option[V] = items.find(p) - def forall(p: V => Boolean): Boolean = items.forall(p) - def foreach[U](f: V => U): Unit = items.foreach(f) - def hasDefiniteSize: Boolean = items.hasDefiniteSize - def isEmpty: Boolean = items.isEmpty - def seq: scala.collection.TraversableOnce[V] = items - def toTraversable: Traversable[V] = items.toTraversable - - override def hashCode() = items.map(_.hashCode()).sum - override def equals(other: Any) = other match{ - case s: Agg[_] => items.sameElements(s.items) - case _ => super.equals(other) - } - override def toString = items.mkString("Agg(", ", ", ")") - } - } -} diff --git a/main/core/src/util/MultiBiMap.scala b/main/core/src/util/MultiBiMap.scala index 73bb42c4..51ea63f2 100644 --- a/main/core/src/util/MultiBiMap.scala +++ b/main/core/src/util/MultiBiMap.scala @@ -1,7 +1,7 @@ package mill.util import scala.collection.mutable -import Strict.Agg +import mill.api.Strict.Agg /** * A map from keys to collections of values: you can assign multiple values diff --git a/main/src/main/ReplApplyHandler.scala b/main/src/main/ReplApplyHandler.scala index a8e467d4..786a1409 100644 --- a/main/src/main/ReplApplyHandler.scala +++ b/main/src/main/ReplApplyHandler.scala @@ -6,7 +6,7 @@ import mill.define.Segment.Label import mill.define._ import mill.eval.{Evaluator, Result} -import mill.util.Strict.Agg +import mill.api.Strict.Agg import scala.collection.mutable object ReplApplyHandler{ diff --git a/main/src/main/RunScript.scala b/main/src/main/RunScript.scala index 47526631..b858c8b9 100644 --- a/main/src/main/RunScript.scala +++ b/main/src/main/RunScript.scala @@ -11,7 +11,7 @@ import mill.define._ import mill.eval.{Evaluator, PathRef, Result} import mill.util.{EitherOps, ParseArgs, Watched} import mill.api.Logger -import mill.util.Strict.Agg +import mill.api.Strict.Agg import scala.collection.mutable import scala.reflect.ClassTag diff --git a/main/src/modules/Jvm.scala b/main/src/modules/Jvm.scala index 1a51ed8b..e17631e3 100644 --- a/main/src/modules/Jvm.scala +++ b/main/src/modules/Jvm.scala @@ -15,7 +15,7 @@ import mill.main.client.InputPumper import mill.eval.{PathRef, Result} import mill.util.Ctx import mill.api.IO -import mill.util.Loose.Agg +import mill.api.Loose.Agg import scala.collection.mutable import scala.collection.JavaConverters._ diff --git a/main/src/modules/Util.scala b/main/src/modules/Util.scala index 2b98a304..029626fe 100644 --- a/main/src/modules/Util.scala +++ b/main/src/modules/Util.scala @@ -3,7 +3,8 @@ package mill.modules import coursier.Repository import mill.api.{PathRef, IO} -import mill.util.{Ctx, Loose} +import mill.util.Ctx +import mill.api.Loose object Util { @@ -55,7 +56,7 @@ object Util { val localPath = sys.props(key) if (localPath != null) { mill.api.Result.Success( - Loose.Agg.from(localPath.split(',').map(p => PathRef(os.Path(p), quick = true))) + mill.api.Loose.Agg.from(localPath.split(',').map(p => PathRef(os.Path(p), quick = true))) ) } else { mill.modules.Jvm.resolveDependencies( diff --git a/main/src/package.scala b/main/src/package.scala index 0ccd094f..6bcb1bdf 100644 --- a/main/src/package.scala +++ b/main/src/package.scala @@ -7,6 +7,6 @@ package object mill extends JsonFormatters{ type PathRef = mill.api.PathRef type Module = define.Module type Cross[T] = define.Cross[T] - type Agg[T] = util.Loose.Agg[T] - val Agg = util.Loose.Agg + type Agg[T] = mill.api.Loose.Agg[T] + val Agg = mill.api.Loose.Agg } diff --git a/main/test/resources/examples/javac/build.sc b/main/test/resources/examples/javac/build.sc index 2ed9f915..17366219 100644 --- a/main/test/resources/examples/javac/build.sc +++ b/main/test/resources/examples/javac/build.sc @@ -2,7 +2,7 @@ import mill.T import mill.eval.JavaCompileJarTests.compileAll import mill.api.PathRef import mill.modules.Jvm -import mill.util.Loose +import mill.api.Loose def sourceRootPath = millSourcePath / 'src def resourceRootPath = millSourcePath / 'resources diff --git a/main/test/src/define/CacherTests.scala b/main/test/src/define/CacherTests.scala index 59ebf3f6..1524e5c1 100644 --- a/main/test/src/define/CacherTests.scala +++ b/main/test/src/define/CacherTests.scala @@ -1,7 +1,7 @@ package mill.define import mill.util.{DummyLogger, TestEvaluator, TestUtil} -import mill.util.Strict.Agg +import mill.api.Strict.Agg import mill.T import mill.api.Result.Success import utest._ diff --git a/main/test/src/define/GraphTests.scala b/main/test/src/define/GraphTests.scala index 224ce59f..b36dbf95 100644 --- a/main/test/src/define/GraphTests.scala +++ b/main/test/src/define/GraphTests.scala @@ -4,7 +4,7 @@ package mill.define import mill.eval.Evaluator import mill.util.{TestGraphs, TestUtil} import utest._ -import mill.util.Strict.Agg +import mill.api.Strict.Agg object GraphTests extends TestSuite{ val tests = Tests{ diff --git a/main/test/src/eval/EvaluationTests.scala b/main/test/src/eval/EvaluationTests.scala index 74f9088c..7f924db2 100644 --- a/main/test/src/eval/EvaluationTests.scala +++ b/main/test/src/eval/EvaluationTests.scala @@ -5,7 +5,7 @@ import mill.util.TestUtil.{Test, test} import mill.define.{Discover, Graph, Target, Task} import mill.{Module, T} import mill.util.{DummyLogger, TestEvaluator, TestGraphs, TestUtil} -import mill.util.Strict.Agg +import mill.api.Strict.Agg import utest._ import utest.framework.TestPath diff --git a/main/test/src/eval/JavaCompileJarTests.scala b/main/test/src/eval/JavaCompileJarTests.scala index 426c6ea6..0f9002df 100644 --- a/main/test/src/eval/JavaCompileJarTests.scala +++ b/main/test/src/eval/JavaCompileJarTests.scala @@ -4,12 +4,13 @@ import mill.define.{Discover, Input, Target, Task} import mill.modules.Jvm import mill.api.Ctx.Dest import mill.{Module, T} -import mill.util.{DummyLogger, Loose, TestEvaluator, TestUtil} -import mill.util.Strict.Agg +import mill.util.{DummyLogger, TestEvaluator, TestUtil} +import mill.api.Strict.Agg +import mill.api.Loose import utest._ import mill._ object JavaCompileJarTests extends TestSuite{ - def compileAll(sources: mill.util.Loose.Agg[PathRef])(implicit ctx: Dest) = { + def compileAll(sources: mill.api.Loose.Agg[PathRef])(implicit ctx: Dest) = { os.makeDir.all(ctx.dest) os.proc("javac", sources.map(_.path.toString()).toSeq, "-d", ctx.dest).call(ctx.dest) diff --git a/main/test/src/util/TestEvaluator.scala b/main/test/src/util/TestEvaluator.scala index 9a235679..81c8fe12 100644 --- a/main/test/src/util/TestEvaluator.scala +++ b/main/test/src/util/TestEvaluator.scala @@ -3,7 +3,7 @@ package mill.util import mill.define.{Input, Target, Task} import mill.api.Result.OuterStack import mill.eval.{Evaluator, Result} -import mill.util.Strict.Agg +import mill.api.Strict.Agg import utest.assert import utest.framework.TestPath diff --git a/main/test/src/util/TestUtil.scala b/main/test/src/util/TestUtil.scala index baab2992..462b7f5c 100644 --- a/main/test/src/util/TestUtil.scala +++ b/main/test/src/util/TestUtil.scala @@ -5,7 +5,7 @@ import mill.define._ import mill.api.Result import mill.api.Result.OuterStack import utest.assert -import mill.util.Strict.Agg +import mill.api.Strict.Agg import utest.framework.TestPath import scala.collection.mutable diff --git a/scalajslib/src/ScalaJSModule.scala b/scalajslib/src/ScalaJSModule.scala index 8568c39b..137e8ee2 100644 --- a/scalajslib/src/ScalaJSModule.scala +++ b/scalajslib/src/ScalaJSModule.scala @@ -7,7 +7,8 @@ import mill.eval.{PathRef, Result} import mill.api.Result.Success import mill.scalalib.Lib.resolveDependencies import mill.scalalib.{DepSyntax, Lib, TestModule, TestRunner} -import mill.util.{Ctx, Loose} +import mill.util.Ctx +import mill.api.Loose import mill.scalajslib.api._ trait ScalaJSModule extends scalalib.ScalaModule { outer => diff --git a/scalalib/api/src/ZincWorkerApi.scala b/scalalib/api/src/ZincWorkerApi.scala index c5230ec5..d42be9f3 100644 --- a/scalalib/api/src/ZincWorkerApi.scala +++ b/scalalib/api/src/ZincWorkerApi.scala @@ -3,14 +3,16 @@ package mill.scalalib.api import mill.api.Loose.Agg import mill.api.PathRef import mill.api.JsonFormatters._ - +object ZincWorkerApi{ + type Ctx = mill.api.Ctx.Dest with mill.api.Ctx.Log with mill.api.Ctx.Home +} trait ZincWorkerApi { /** Compile a Java-only project */ def compileJava(upstreamCompileOutput: Seq[CompilationResult], sources: Agg[os.Path], compileClasspath: Agg[os.Path], javacOptions: Seq[String]) - (implicit ctx: mill.api.Ctx): mill.api.Result[CompilationResult] + (implicit ctx: ZincWorkerApi.Ctx): mill.api.Result[CompilationResult] /** Compile a mixed Scala/Java or Scala-only project */ def compileMixed(upstreamCompileOutput: Seq[CompilationResult], @@ -18,21 +20,21 @@ trait ZincWorkerApi { compileClasspath: Agg[os.Path], javacOptions: Seq[String], scalaVersion: String, + scalaOrganization: String, scalacOptions: Seq[String], - compilerBridgeSources: os.Path, compilerClasspath: Agg[os.Path], scalacPluginClasspath: Agg[os.Path]) - (implicit ctx: mill.api.Ctx): mill.api.Result[CompilationResult] + (implicit ctx: ZincWorkerApi.Ctx): mill.api.Result[CompilationResult] def discoverMainClasses(compilationResult: CompilationResult) - (implicit ctx: mill.api.Ctx): Seq[String] + (implicit ctx: ZincWorkerApi.Ctx): Seq[String] def docJar(scalaVersion: String, - compilerBridgeSources: os.Path, + scalaOrganization: String, compilerClasspath: Agg[os.Path], scalacPluginClasspath: Agg[os.Path], args: Seq[String]) - (implicit ctx: mill.api.Ctx): Boolean + (implicit ctx: ZincWorkerApi.Ctx): Boolean } diff --git a/scalalib/src/GenIdeaImpl.scala b/scalalib/src/GenIdeaImpl.scala index 548b8d4e..b8f9d35e 100644 --- a/scalalib/src/GenIdeaImpl.scala +++ b/scalalib/src/GenIdeaImpl.scala @@ -5,8 +5,8 @@ import coursier.{Cache, CoursierPaths, Repository} import mill.define._ import mill.eval.{Evaluator, PathRef, Result} import mill.api.Ctx.{Home, Log} -import mill.util.Strict.Agg -import mill.util.{Loose, Strict} +import mill.api.Strict.Agg +import mill.api.{Loose, Strict} import mill.{T, scalalib} import scala.util.Try diff --git a/scalalib/src/JavaModule.scala b/scalalib/src/JavaModule.scala index 78be8893..72c0a5a6 100644 --- a/scalalib/src/JavaModule.scala +++ b/scalalib/src/JavaModule.scala @@ -9,7 +9,7 @@ import mill.modules.{Assembly, Jvm} import mill.modules.Jvm.{createAssembly, createJar} import Lib._ import mill.scalalib.publish.{Artifact, Scope} -import mill.util.Loose.Agg +import mill.api.Loose.Agg /** * Core configuration required to compile a single Scala compilation target diff --git a/scalalib/src/MiscModule.scala b/scalalib/src/MiscModule.scala index c6449d6e..bf64f1f3 100644 --- a/scalalib/src/MiscModule.scala +++ b/scalalib/src/MiscModule.scala @@ -4,7 +4,7 @@ package scalalib import mill.define.Cross.Resolver import mill.define.{Cross, Task} import mill.eval.{PathRef, Result} -import mill.util.Loose.Agg +import mill.api.Loose.Agg object CrossModuleBase{ def scalaVersionPaths(scalaVersion: String, f: String => os.Path) = { for(segments <- scalaVersion.split('.').inits.filter(_.nonEmpty)) diff --git a/scalalib/src/ScalaModule.scala b/scalalib/src/ScalaModule.scala index 9d669bf4..5fad1664 100644 --- a/scalalib/src/ScalaModule.scala +++ b/scalalib/src/ScalaModule.scala @@ -8,7 +8,7 @@ import mill.modules.Jvm import mill.modules.Jvm.createJar import mill.scalalib.api.Util.isDotty import Lib._ -import mill.util.Loose.Agg +import mill.api.Loose.Agg import mill.api.DummyInputStream /** @@ -79,36 +79,7 @@ trait ScalaModule extends JavaModule { outer => def scalaDocOptions = T{ scalacOptions() } - private val Milestone213 = raw"""2.13.(\d+)-M(\d+)""".r - def scalaCompilerBridgeSources = T { - val (scalaVersion0, scalaBinaryVersion0) = scalaVersion() match { - case Milestone213(_, _) => ("2.13.0-M2", "2.13.0-M2") - case _ => (scalaVersion(), mill.scalalib.api.Util.scalaBinaryVersion(scalaVersion())) - } - - val (bridgeDep, bridgeName, bridgeVersion) = - if (isDotty(scalaVersion0)) { - val org = scalaOrganization() - val name = "dotty-sbt-bridge" - val version = scalaVersion() - (ivy"$org:$name:$version", name, version) - } else { - val org = "org.scala-sbt" - val name = "compiler-bridge" - val version = Versions.zinc - (ivy"$org::$name:$version", s"${name}_$scalaBinaryVersion0", version) - } - - resolveDependencies( - repositories, - Lib.depToDependency(_, scalaVersion0, platformSuffix()), - Seq(bridgeDep), - sources = true - ).map(deps => - mill.scalalib.api.Util.grepJar(deps.map(_.path), bridgeName, bridgeVersion, sources = true) - ) - } /** * The local classpath of Scala compiler plugins on-disk; you can add @@ -159,8 +130,8 @@ trait ScalaModule extends JavaModule { outer => compileClasspath().map(_.path), javacOptions(), scalaVersion(), + scalaOrganization(), scalacOptions(), - scalaCompilerBridgeSources(), scalaCompilerClasspath().map(_.path), scalacPluginClasspath().map(_.path), ) @@ -187,7 +158,7 @@ trait ScalaModule extends JavaModule { outer => else { zincWorker.worker().docJar( scalaVersion(), - scalaCompilerBridgeSources(), + scalaOrganization(), scalaCompilerClasspath().map(_.path), scalacPluginClasspath().map(_.path), files ++ options diff --git a/scalalib/src/ZincWorkerModule.scala b/scalalib/src/ZincWorkerModule.scala index 5ca824ce..97d84aaf 100644 --- a/scalalib/src/ZincWorkerModule.scala +++ b/scalalib/src/ZincWorkerModule.scala @@ -4,9 +4,12 @@ import coursier.Cache import coursier.maven.MavenRepository import mill.Agg import mill.T +import mill.api.KeyedLockedCache import mill.define.{Discover, Worker} import mill.scalalib.Lib.resolveDependencies -import mill.util.Loose +import mill.scalalib.api.Util.isDotty +import mill.scalalib.api.ZincWorkerApi +import mill.api.Loose import mill.util.JsonFormatters._ object ZincWorkerModule extends mill.define.ExternalModule with ZincWorkerModule{ @@ -40,11 +43,61 @@ trait ZincWorkerModule extends mill.Module{ getClass.getClassLoader ) val cls = cl.loadClass("mill.scalalib.worker.ZincWorkerImpl") - val instance = cls.getConstructor(classOf[mill.api.Ctx], classOf[Array[String]]) - .newInstance(T.ctx(), compilerInterfaceClasspath().map(_.path.toString).toArray[String]) + val instance = cls.getConstructor( + classOf[ + Either[ + (ZincWorkerApi.Ctx, Array[os.Path], (String, String) => os.Path), + String => os.Path + ] + ], + classOf[(Agg[os.Path], String) => os.Path], + classOf[(Agg[os.Path], String) => os.Path], + classOf[KeyedLockedCache[_]] + ) + .newInstance( + Left(( + T.ctx(), + compilerInterfaceClasspath().map(_.path).toArray, + (x: String, y: String) => scalaCompilerBridgeSourceJar(x, y).asSuccess.get.value + )), + mill.scalalib.api.Util.grepJar(_, "scala-library", _, sources = false), + mill.scalalib.api.Util.grepJar(_, "scala-compiler", _, sources = false), + new KeyedLockedCache.RandomBoundedCache(1, 1) + ) instance.asInstanceOf[mill.scalalib.api.ZincWorkerApi] } + private val Milestone213 = raw"""2.13.(\d+)-M(\d+)""".r + def scalaCompilerBridgeSourceJar(scalaVersion: String, + scalaOrganization: String) = { + val (scalaVersion0, scalaBinaryVersion0) = scalaVersion match { + case Milestone213(_, _) => ("2.13.0-M2", "2.13.0-M2") + case _ => (scalaVersion, mill.scalalib.api.Util.scalaBinaryVersion(scalaVersion)) + } + + val (bridgeDep, bridgeName, bridgeVersion) = + if (isDotty(scalaVersion0)) { + val org = scalaOrganization + val name = "dotty-sbt-bridge" + val version = scalaVersion + (ivy"$org:$name:$version", name, version) + } else { + val org = "org.scala-sbt" + val name = "compiler-bridge" + val version = Versions.zinc + (ivy"$org::$name:$version", s"${name}_$scalaBinaryVersion0", version) + } + + resolveDependencies( + repositories, + Lib.depToDependency(_, scalaVersion0, ""), + Seq(bridgeDep), + sources = true + ).map(deps => + mill.scalalib.api.Util.grepJar(deps.map(_.path), bridgeName, bridgeVersion, sources = true) + ) + } + def compilerInterfaceClasspath = T{ resolveDependencies( repositories, diff --git a/scalalib/src/dependency/versions/VersionsFinder.scala b/scalalib/src/dependency/versions/VersionsFinder.scala index a831ffc3..a9ecc763 100644 --- a/scalalib/src/dependency/versions/VersionsFinder.scala +++ b/scalalib/src/dependency/versions/VersionsFinder.scala @@ -5,7 +5,7 @@ import mill.eval.Evaluator import mill.scalalib.dependency.metadata.MetadataLoaderFactory import mill.scalalib.{Dep, JavaModule, Lib} import mill.api.Ctx.{Home, Log} -import mill.util.{Loose, Strict} +import mill.api.{Loose, Strict} private[dependency] object VersionsFinder { diff --git a/scalalib/src/publish/Ivy.scala b/scalalib/src/publish/Ivy.scala index 22e26ff6..e06efadd 100644 --- a/scalalib/src/publish/Ivy.scala +++ b/scalalib/src/publish/Ivy.scala @@ -1,6 +1,6 @@ package mill.scalalib.publish -import mill.util.Loose.Agg +import mill.api.Loose.Agg import scala.xml.PrettyPrinter diff --git a/scalalib/src/publish/Pom.scala b/scalalib/src/publish/Pom.scala index 57a0e196..a7f1f6fc 100644 --- a/scalalib/src/publish/Pom.scala +++ b/scalalib/src/publish/Pom.scala @@ -1,6 +1,6 @@ package mill.scalalib.publish -import mill.util.Loose.Agg +import mill.api.Loose.Agg import scala.xml.{Atom, Elem, NodeSeq, PrettyPrinter} diff --git a/scalalib/test/src/ResolveDepsTests.scala b/scalalib/test/src/ResolveDepsTests.scala index 78361625..ce905907 100644 --- a/scalalib/test/src/ResolveDepsTests.scala +++ b/scalalib/test/src/ResolveDepsTests.scala @@ -4,7 +4,7 @@ import coursier.Cache import coursier.maven.MavenRepository import mill.api.Result.{Failure, Success} import mill.eval.{PathRef, Result} -import mill.util.Loose.Agg +import mill.api.Loose.Agg import utest._ object ResolveDepsTests extends TestSuite { diff --git a/scalalib/worker/src/ZincWorkerImpl.scala b/scalalib/worker/src/ZincWorkerImpl.scala index 705d4682..c37ef162 100644 --- a/scalalib/worker/src/ZincWorkerImpl.scala +++ b/scalalib/worker/src/ZincWorkerImpl.scala @@ -4,13 +4,13 @@ import java.io.File import java.util.Optional import mill.api.Loose.Agg -import mill.api.PathRef +import mill.api.{KeyedLockedCache, PathRef} import xsbti.compile.{CompilerCache => _, FileAnalysisStore => _, ScalaInstance => _, _} -import mill.scalalib.api.Util.{isDotty, grepJar, scalaBinaryVersion} +import mill.scalalib.api.Util.{grepJar, isDotty, scalaBinaryVersion} import sbt.internal.inc._ import sbt.internal.util.{ConsoleOut, MainAppender} import sbt.util.LogExchange -import mill.scalalib.api.CompilationResult +import mill.scalalib.api.{CompilationResult, ZincWorkerApi} case class MockedLookup(am: File => Optional[CompileAnalysis]) extends PerClasspathEntryLookup { override def analysis(classpathEntry: File): Optional[CompileAnalysis] = am(classpathEntry) @@ -19,10 +19,16 @@ case class MockedLookup(am: File => Optional[CompileAnalysis]) extends PerClassp Locate.definesClass(classpathEntry) } -class ZincWorkerImpl(ctx0: mill.api.Ctx, - compilerBridgeClasspath: Array[String]) extends mill.scalalib.api.ZincWorkerApi{ +class ZincWorkerImpl(compilerBridge: Either[ + (ZincWorkerApi.Ctx, Array[os.Path], (String, String) => os.Path), + String => os.Path + ], + libraryJarNameGrep: (Agg[os.Path], String) => os.Path, + compilerJarNameGrep: (Agg[os.Path], String) => os.Path, + compilerCache: KeyedLockedCache[Compilers]) + extends ZincWorkerApi{ private val ic = new sbt.internal.inc.IncrementalCompilerImpl() - val javaOnlyCompilers = { + lazy val javaOnlyCompilers = { // Keep the classpath as written by the user val classpathOptions = ClasspathOptions.of(false, false, false, false, false) @@ -42,68 +48,68 @@ class ZincWorkerImpl(ctx0: mill.api.Ctx, ) } - @volatile var mixedCompilersCache = Option.empty[(Long, Compilers)] - def docJar(scalaVersion: String, - compilerBridgeSources: os.Path, + scalaOrganization: String, compilerClasspath: Agg[os.Path], scalacPluginClasspath: Agg[os.Path], args: Seq[String]) - (implicit ctx: mill.api.Ctx): Boolean = { - val compilers: Compilers = prepareCompilers( + (implicit ctx: ZincWorkerApi.Ctx): Boolean = { + withCompilers( scalaVersion, - compilerBridgeSources, + scalaOrganization, compilerClasspath, - scalacPluginClasspath - ) - val scaladocClass = compilers.scalac().scalaInstance().loader().loadClass("scala.tools.nsc.ScalaDoc") - val scaladocMethod = scaladocClass.getMethod("process", classOf[Array[String]]) - scaladocMethod.invoke(scaladocClass.newInstance(), args.toArray).asInstanceOf[Boolean] + scalacPluginClasspath, + ) { compilers: Compilers => + val scaladocClass = compilers.scalac().scalaInstance().loader().loadClass("scala.tools.nsc.ScalaDoc") + val scaladocMethod = scaladocClass.getMethod("process", classOf[Array[String]]) + scaladocMethod.invoke(scaladocClass.newInstance(), args.toArray).asInstanceOf[Boolean] + } } /** Compile the bridge if it doesn't exist yet and return the output directory. - * TODO: Proper invalidation, see #389 - */ - def compileZincBridgeIfNeeded(scalaVersion: String, - sourcesJar: os.Path, - compilerJars: Array[File]): os.Path = { - val workingDir = ctx0.dest / scalaVersion - val compiledDest = workingDir / 'compiled - if (!os.exists(workingDir)) { - - ctx0.log.info("Compiling compiler interface...") - - os.makeDir.all(workingDir) - os.makeDir.all(compiledDest) - - val sourceFolder = mill.api.IO.unpackZip(sourcesJar)(workingDir) - val classloader = mill.api.ClassLoader.create(compilerJars.map(_.toURI.toURL), null)(ctx0) - val compilerMain = classloader.loadClass( - if (isDotty(scalaVersion)) - "dotty.tools.dotc.Main" - else - "scala.tools.nsc.Main" - ) - val argsArray = Array[String]( - "-d", compiledDest.toString, - "-classpath", (compilerJars ++ compilerBridgeClasspath).mkString(File.pathSeparator) - ) ++ os.walk(sourceFolder.path).filter(_.ext == "scala").map(_.toString) - - compilerMain.getMethod("process", classOf[Array[String]]) - .invoke(null, argsArray) + * TODO: Proper invalidation, see #389 + */ + def compileZincBridgeIfNeeded(scalaVersion: String, scalaOrganization: String, compilerJars: Array[File]): os.Path = { + compilerBridge match{ + case Right(compiled) => compiled(scalaVersion) + case Left((ctx0, compilerBridgeClasspath, srcJars)) => + val workingDir = ctx0.dest / scalaVersion + val compiledDest = workingDir / 'compiled + if (!os.exists(workingDir)) { + ctx0.log.info("Compiling compiler interface...") + + os.makeDir.all(workingDir) + os.makeDir.all(compiledDest) + + val sourceFolder = mill.api.IO.unpackZip(srcJars(scalaVersion, scalaOrganization))(workingDir) + val classloader = mill.api.ClassLoader.create(compilerJars.map(_.toURI.toURL), null)(ctx0) + val compilerMain = classloader.loadClass( + if (isDotty(scalaVersion)) "dotty.tools.dotc.Main" + else "scala.tools.nsc.Main" + ) + val argsArray = Array[String]( + "-d", compiledDest.toString, + "-classpath", (compilerJars ++ compilerBridgeClasspath).mkString(File.pathSeparator) + ) ++ os.walk(sourceFolder.path).filter(_.ext == "scala").map(_.toString) + + compilerMain.getMethod("process", classOf[Array[String]]) + .invoke(null, argsArray) + } + compiledDest } - compiledDest + } - def discoverMainClasses(compilationResult: CompilationResult)(implicit ctx: mill.api.Ctx): Seq[String] = { + def discoverMainClasses(compilationResult: CompilationResult) + (implicit ctx: ZincWorkerApi.Ctx): Seq[String] = { def toScala[A](o: Optional[A]): Option[A] = if (o.isPresent) Some(o.get) else None toScala(FileAnalysisStore.binary(compilationResult.analysisFile.toIO).get()) .map(_.getAnalysis) .flatMap{ case analysis: Analysis => - Some(analysis.infos.allInfos.values.map(_.getMainClasses).flatten.toSeq.sorted) + Some(analysis.infos.allInfos.values.flatMap(_.getMainClasses).toSeq.sorted) case _ => None } @@ -114,7 +120,7 @@ class ZincWorkerImpl(ctx0: mill.api.Ctx, sources: Agg[os.Path], compileClasspath: Agg[os.Path], javacOptions: Seq[String]) - (implicit ctx: mill.api.Ctx): mill.api.Result[CompilationResult] = { + (implicit ctx: ZincWorkerApi.Ctx): mill.api.Result[CompilationResult] = { compileInternal( upstreamCompileOutput, sources, @@ -130,73 +136,70 @@ class ZincWorkerImpl(ctx0: mill.api.Ctx, compileClasspath: Agg[os.Path], javacOptions: Seq[String], scalaVersion: String, + scalaOrganization: String, scalacOptions: Seq[String], - compilerBridgeSources: os.Path, compilerClasspath: Agg[os.Path], scalacPluginClasspath: Agg[os.Path]) - (implicit ctx: mill.api.Ctx): mill.api.Result[CompilationResult] = { - val compilers: Compilers = prepareCompilers( + (implicit ctx: ZincWorkerApi.Ctx): mill.api.Result[CompilationResult] = { + withCompilers( scalaVersion, - compilerBridgeSources, + scalaOrganization, compilerClasspath, - scalacPluginClasspath - ) - - compileInternal( - upstreamCompileOutput, - sources, - compileClasspath, - javacOptions, - scalacOptions = scalacPluginClasspath.map(jar => s"-Xplugin:${jar}").toSeq ++ scalacOptions, - compilers - ) + scalacPluginClasspath, + ) {compilers: Compilers => + compileInternal( + upstreamCompileOutput, + sources, + compileClasspath, + javacOptions, + scalacOptions = scalacPluginClasspath.map(jar => s"-Xplugin:$jar").toSeq ++ scalacOptions, + compilers + ) + } } - private def prepareCompilers(scalaVersion: String, - compilerBridgeSources: os.Path, + private def withCompilers[T](scalaVersion: String, + scalaOrganization: String, compilerClasspath: Agg[os.Path], scalacPluginClasspath: Agg[os.Path]) - (implicit ctx: mill.api.Ctx)= { + (f: Compilers => T) + (implicit ctx: ZincWorkerApi.Ctx)= { val combinedCompilerClasspath = compilerClasspath ++ scalacPluginClasspath val combinedCompilerJars = combinedCompilerClasspath.toArray.map(_.toIO) - val compilerBridge = compileZincBridgeIfNeeded( + val compiledCompilerBridge = compileZincBridgeIfNeeded( scalaVersion, - compilerBridgeSources, + scalaOrganization, compilerClasspath.toArray.map(_.toIO) ) - val compilerBridgeSig = os.mtime(compilerBridge) + + val compilerBridgeSig = os.mtime(compiledCompilerBridge) val compilersSig = compilerBridgeSig + combinedCompilerClasspath.map(p => p.toString().hashCode + os.mtime(p)).sum - val compilers = mixedCompilersCache match { - case Some((k, v)) if k == compilersSig => v - case _ => - val compilerName = - if (isDotty(scalaVersion)) - s"dotty-compiler_${scalaBinaryVersion(scalaVersion)}" - else - "scala-compiler" - val scalaInstance = new ScalaInstance( - version = scalaVersion, - loader = mill.api.ClassLoader.create(combinedCompilerJars.map(_.toURI.toURL), null), - libraryJar = grepJar(compilerClasspath, "scala-library", scalaVersion).toIO, - compilerJar = grepJar(compilerClasspath, compilerName, scalaVersion).toIO, - allJars = combinedCompilerJars, - explicitActual = None - ) - val compilers = ic.compilers( - scalaInstance, - ClasspathOptionsUtil.boot, - None, - ZincUtil.scalaCompiler(scalaInstance, compilerBridge.toIO) - ) - mixedCompilersCache = Some((compilersSig, compilers)) - compilers - } - compilers + compilerCache.withCachedValue(compilersSig){ + val compilerJar = + if (isDotty(scalaVersion)) + grepJar(compilerClasspath, s"dotty-compiler_${scalaBinaryVersion(scalaVersion)}", scalaVersion) + else + compilerJarNameGrep(compilerClasspath, scalaVersion) + val scalaInstance = new ScalaInstance( + version = scalaVersion, + loader = mill.api.ClassLoader.create(combinedCompilerJars.map(_.toURI.toURL), null), + libraryJar = libraryJarNameGrep(compilerClasspath, scalaVersion).toIO, + compilerJar = compilerJar.toIO, + allJars = combinedCompilerJars, + explicitActual = None + ) + ic.compilers( + scalaInstance, + ClasspathOptionsUtil.boot, + None, + ZincUtil.scalaCompiler(scalaInstance, compiledCompilerBridge.toIO) + ) + }(f) } private def compileInternal(upstreamCompileOutput: Seq[CompilationResult], @@ -205,7 +208,7 @@ class ZincWorkerImpl(ctx0: mill.api.Ctx, javacOptions: Seq[String], scalacOptions: Seq[String], compilers: Compilers) - (implicit ctx: mill.api.Ctx): mill.api.Result[CompilationResult] = { + (implicit ctx: ZincWorkerApi.Ctx): mill.api.Result[CompilationResult] = { os.makeDir.all(ctx.dest) val logger = { diff --git a/scalanativelib/src/ScalaNativeModule.scala b/scalanativelib/src/ScalaNativeModule.scala index a7a2b96b..289ba759 100644 --- a/scalanativelib/src/ScalaNativeModule.scala +++ b/scalanativelib/src/ScalaNativeModule.scala @@ -9,7 +9,7 @@ import mill.define.{Target, Task} import mill.api.Result import mill.modules.Jvm import mill.scalalib.{Dep, DepSyntax, Lib, SbtModule, ScalaModule, TestModule, TestRunner} -import mill.util.Loose.Agg +import mill.api.Loose.Agg import sbt.testing.{AnnotatedFingerprint, SubclassFingerprint} import sbt.testing.Fingerprint import upickle.default.{ReadWriter => RW, macroRW} |