diff options
author | Li Haoyi <haoyi.sg@gmail.com> | 2018-01-23 01:38:20 -0800 |
---|---|---|
committer | Li Haoyi <haoyi.sg@gmail.com> | 2018-01-23 01:50:56 -0800 |
commit | 0ea8b3d10ae7500426b174a33ef70d03d474ecc4 (patch) | |
tree | ee18b2e3605dd3c71c0d00d9eed7383b4a38529c /core | |
parent | 4a0658da074bc7b7df0c5bdff90e2c6bb1977b15 (diff) | |
download | mill-0ea8b3d10ae7500426b174a33ef70d03d474ecc4.tar.gz mill-0ea8b3d10ae7500426b174a33ef70d03d474ecc4.tar.bz2 mill-0ea8b3d10ae7500426b174a33ef70d03d474ecc4.zip |
WIP splitting `mill.scalaworker` out of `mill.scalalib` and into it's own isolated module/classloader. Most scalalib test pass, tho GenIdea is still broken
Diffstat (limited to 'core')
-rw-r--r-- | core/src/mill/define/Module.scala | 5 | ||||
-rw-r--r-- | core/src/mill/define/Task.scala | 26 | ||||
-rw-r--r-- | core/src/mill/define/Worker.scala | 32 | ||||
-rw-r--r-- | core/src/mill/eval/Evaluator.scala | 60 | ||||
-rw-r--r-- | core/src/mill/main/MainRunner.scala | 2 | ||||
-rw-r--r-- | core/src/mill/modules/Jvm.scala | 9 | ||||
-rw-r--r-- | core/src/mill/modules/Util.scala | 2 |
7 files changed, 74 insertions, 62 deletions
diff --git a/core/src/mill/define/Module.scala b/core/src/mill/define/Module.scala index 222bb7ec..dd451c9c 100644 --- a/core/src/mill/define/Module.scala +++ b/core/src/mill/define/Module.scala @@ -89,7 +89,9 @@ object Module{ trait TaskModule extends Module { def defaultCommandName(): String } - +object BaseModule{ + case class Implicit(value: BaseModule) +} class BaseModule(basePath0: Path) (implicit millModuleEnclosing0: sourcecode.Enclosing, millModuleLine0: sourcecode.Line, @@ -104,4 +106,5 @@ class BaseModule(basePath0: Path) override implicit def millModuleSegments: Segments = Segments() override implicit def millModuleBasePath: BasePath = BasePath(millOuterCtx.basePath) override def basePath = millOuterCtx.basePath + implicit def millImplicitBaseModule: BaseModule.Implicit = BaseModule.Implicit(this) }
\ No newline at end of file diff --git a/core/src/mill/define/Task.scala b/core/src/mill/define/Task.scala index 248f145c..bafabd01 100644 --- a/core/src/mill/define/Task.scala +++ b/core/src/mill/define/Task.scala @@ -37,6 +37,7 @@ abstract class Task[+T] extends Task.Ops[T] with Applyable[Task, T]{ def asTarget: Option[Target[T]] = None def asCommand: Option[Command[T]] = None def asPersistent: Option[Persistent[T]] = None + def asWorker: Option[Worker[T]] = None def self = this } @@ -179,6 +180,21 @@ object Target extends TargetGenerated with Applicative.Applyer[Task, Task, Resul ) } + def worker[T](t: Task[T]) + (implicit ctx: mill.define.Ctx): Worker[T] = new Worker(t, ctx) + + def worker[T](t: Result[T]) + (implicit ctx: mill.define.Ctx): Worker[T] = macro workerImpl[T] + + def workerImpl[T: c.WeakTypeTag](c: Context) + (t: c.Expr[T]) + (ctx: c.Expr[mill.define.Ctx]): c.Expr[Worker[T]] = { + import c.universe._ + reify( + new Worker[T](Applicative.impl[Task, T, mill.util.Ctx](c)(t).splice, ctx.splice) + ) + } + def task[T](t: Result[T]): Task[T] = macro Applicative.impl[Task, T, mill.util.Ctx] def persistent[T](t: Result[T])(implicit r: R[T], @@ -233,8 +249,8 @@ class TargetImpl[+T](t: Task[T], val ctx = ctx0.copy(segments = ctx0.segments ++ Seq(ctx0.segment)) val inputs = Seq(t) def evaluate(args: mill.util.Ctx) = args[T](0) - } + class Command[+T](t: Task[T], ctx0: mill.define.Ctx, val writer: W[_]) extends NamedTask[T] { @@ -243,6 +259,14 @@ class Command[+T](t: Task[T], def evaluate(args: mill.util.Ctx) = args[T](0) override def asCommand = Some(this) } + +class Worker[+T](t: Task[T], + ctx0: mill.define.Ctx) extends NamedTask[T] { + val ctx = ctx0.copy(segments = ctx0.segments ++ Seq(ctx0.segment)) + val inputs = Seq(t) + def evaluate(args: mill.util.Ctx) = args[T](0) + override def asWorker = Some(this) +} class Persistent[+T](t: Task[T], ctx0: mill.define.Ctx, readWrite: RW[_]) diff --git a/core/src/mill/define/Worker.scala b/core/src/mill/define/Worker.scala deleted file mode 100644 index 3d35d2e0..00000000 --- a/core/src/mill/define/Worker.scala +++ /dev/null @@ -1,32 +0,0 @@ -package mill.define - - -/** - * Worker serves three purposes: - * - * - Cache in-memory state between tasks (e.g. object initialization) - * - Including warm classloaders with isolated bytecode - * - Mutex to limit concurrency - * - Manage out-of-process subprocesses <-- skip this for now - * - * Key usage: - * - * - T{ - * ZincWorker().compile(a() + b()) - * } - * - * Desugars into: - * - * - T.zipMap(ZincWorker, a, b){ (z, a1, b1) => z.compile(a1, b1) } - * - * Workers are shoehorned into the `Task` type. This lets them fit nicely in - * the `T{...}` syntax, as well as being statically-inspectable before - * evaluating the task graph. The Worker defines how it is evaluated, but it's - * evaluation/caching/lifecycle are controlled by the `Evaluator` - */ -trait Worker[V] extends Task[V] with mill.util.Ctx.Loader[V]{ - val inputs = Nil - def make(): V - def evaluate(args: mill.util.Ctx) = args.load(this) - def path = this.getClass.getCanonicalName.filter(_ != '$').split('.') -} diff --git a/core/src/mill/eval/Evaluator.scala b/core/src/mill/eval/Evaluator.scala index 44f24275..cf4b32cf 100644 --- a/core/src/mill/eval/Evaluator.scala +++ b/core/src/mill/eval/Evaluator.scala @@ -4,7 +4,7 @@ import java.net.URLClassLoader import ammonite.ops._ import ammonite.runtime.SpecialClassLoader -import mill.define.{Graph, NamedTask, Segment, Segments, Target, Task} +import mill.define.{Ctx => _, _} import mill.util import mill.util.Ctx.Loader import mill.util._ @@ -34,8 +34,7 @@ class Evaluator[T](val workspacePath: Path, val classLoaderSig: Seq[(Path, Long)] = Evaluator.classLoaderSig){ - val workerCache = mutable.Map.empty[Ctx.Loader[_], Any] - workerCache(RootModuleLoader) = rootModule + val workerCache = mutable.Map.empty[Segments, (Int, Any)] def evaluate(goals: Agg[Task[_]]): Evaluator.Results = { mkdir(workspacePath) @@ -47,6 +46,7 @@ class Evaluator[T](val workspacePath: Path, val (finalTaskOverrides, enclosing) = t match{ case t: Target[_] => rootModule.millInternal.segmentsToTargets(segments).ctx.overrides -> t.ctx.enclosing case c: mill.define.Command[_] => 0 -> c.ctx.enclosing + case c: mill.define.Worker[_] => 0 -> c.ctx.enclosing } val additional = if (finalTaskOverrides == t.ctx.overrides) Nil @@ -107,33 +107,44 @@ class Evaluator[T](val workspacePath: Path, maybeTargetLabel = None, counterMsg = counterMsg ) - case Right(labelledTarget) => - val paths = Evaluator.resolveDestPaths(workspacePath, labelledTarget.segments) - val groupBasePath = basePath / Evaluator.makeSegmentStrings(labelledTarget.segments) + case Right(labelledNamedTask) => + val paths = Evaluator.resolveDestPaths(workspacePath, labelledNamedTask.segments) + val groupBasePath = basePath / Evaluator.makeSegmentStrings(labelledNamedTask.segments) mkdir(paths.out) val cached = for{ json <- scala.util.Try(upickle.json.read(read(paths.meta))).toOption (cachedHash, terminalResult) <- scala.util.Try(upickle.default.readJs[(Int, upickle.Js.Value)](json)).toOption if cachedHash == inputsHash - reader <- labelledTarget.format + reader <- labelledNamedTask.format parsed <- reader.read.lift(terminalResult) } yield parsed - cached match{ - case Some(parsed) => + val workerCached = labelledNamedTask.target.asWorker + .flatMap{w => workerCache.get(w.ctx.segments)} + .filter(_._1 == inputsHash) + + (workerCached, cached) match{ + case (Some(workerValue), _) => val newResults = mutable.LinkedHashMap.empty[Task[_], Result[Any]] - newResults(labelledTarget.target) = parsed + newResults(labelledNamedTask.target) = { + Result.Success(workerValue._2) + } + (newResults, Nil) + + case (_, Some(parsed)) => + val newResults = mutable.LinkedHashMap.empty[Task[_], Result[Any]] + newResults(labelledNamedTask.target) = parsed (newResults, Nil) case _ => - val Seq(first, rest @_*) = labelledTarget.segments.value + val Seq(first, rest @_*) = labelledNamedTask.segments.value val msgParts = Seq(first.asInstanceOf[Segment.Label].value) ++ rest.map{ case Segment.Label(s) => "." + s case Segment.Cross(s) => "[" + s.mkString(",") + "]" } - if (labelledTarget.target.flushDest) rm(paths.dest) + if (labelledNamedTask.target.flushDest) rm(paths.dest) val (newResults, newEvaluated) = evaluateGroup( group, results, @@ -143,15 +154,20 @@ class Evaluator[T](val workspacePath: Path, counterMsg = counterMsg ) - newResults(labelledTarget.target) match{ + newResults(labelledNamedTask.target) match{ case Result.Success(v) => - val terminalResult = labelledTarget - .writer - .asInstanceOf[Option[upickle.default.Writer[Any]]] - .map(_.write(v)) - - for(t <- terminalResult){ - write.over(paths.meta, upickle.default.write(inputsHash -> t, indent = 4)) + labelledNamedTask.target.asWorker match{ + case Some(w) => + workerCache(w.ctx.segments) = (inputsHash, v) + case None => + val terminalResult = labelledNamedTask + .writer + .asInstanceOf[Option[upickle.default.Writer[Any]]] + .map(_.write(v)) + + for(t <- terminalResult){ + write.over(paths.meta, upickle.default.write(inputsHash -> t, indent = 4)) + } } case _ => // Wipe out any cached meta.json file that exists, so @@ -211,9 +227,7 @@ class Evaluator[T](val workspacePath: Path, groupBasePath.orNull, multiLogger, new Ctx.LoaderCtx{ - def load[T](x: Ctx.Loader[T]): T = { - workerCache.getOrElseUpdate(x, x.make()).asInstanceOf[T] - } + def load[T](x: Ctx.Loader[T]): T = ??? } ) diff --git a/core/src/mill/main/MainRunner.scala b/core/src/mill/main/MainRunner.scala index 5281b886..d3053d7a 100644 --- a/core/src/mill/main/MainRunner.scala +++ b/core/src/mill/main/MainRunner.scala @@ -101,7 +101,7 @@ class MainRunner(config: ammonite.main.Cli.Config, | val millSelf = Some(this) |} | - |sealed trait $wrapName extends mill.Module{ + |sealed trait $wrapName extends mill.Module{this: mill.define.BaseModule => |""".stripMargin } diff --git a/core/src/mill/modules/Jvm.scala b/core/src/mill/modules/Jvm.scala index 7f2ca4dd..0fa7e3e3 100644 --- a/core/src/mill/modules/Jvm.scala +++ b/core/src/mill/modules/Jvm.scala @@ -9,12 +9,14 @@ import java.util.jar.{JarEntry, JarFile, JarOutputStream} import ammonite.ops._ import mill.define.Task import mill.eval.PathRef -import mill.util.Ctx +import mill.util.{Ctx, Loose} import mill.util.Ctx.LogCtx import mill.util.Loose.Agg +import upickle.default.{Reader, Writer} import scala.annotation.tailrec import scala.collection.mutable +import scala.reflect.ClassTag object Jvm { @@ -71,9 +73,10 @@ object Jvm { } + def inprocess[T](classPath: Agg[Path], - classLoaderOverrideSbtTesting: Boolean, - body: ClassLoader => T): T = { + classLoaderOverrideSbtTesting: Boolean, + body: ClassLoader => T): T = { val cl = if (classLoaderOverrideSbtTesting) { val outerClassLoader = getClass.getClassLoader new URLClassLoader( diff --git a/core/src/mill/modules/Util.scala b/core/src/mill/modules/Util.scala index cd674bad..d53cfcc9 100644 --- a/core/src/mill/modules/Util.scala +++ b/core/src/mill/modules/Util.scala @@ -1,6 +1,6 @@ package mill.modules -import ammonite.ops.RelPath +import ammonite.ops.{Path, RelPath} import mill.eval.PathRef import mill.util.Ctx |