- Convert `T.source` into a generic `Target` that flushes the cache every time

- Prepare `T.ctx().base: Path` that `Task`s (including `T.source`) can use to find a "default" path for source files.

- Simplify `Cacher`

6 files changed, 120 insertions, 61 deletions

diff --git a/core/src/main/scala/mill/define/Task.scala b/core/src/main/scala/mill/define/Task.scala
index b96e80c6..39ecb896 100644
--- a/core/src/main/scala/mill/define/Task.scala
+++ b/core/src/main/scala/mill/define/Task.scala
@@ -66,7 +66,7 @@ object Target extends TargetGenerated with Applicative.Applyer[Task, Task, Resul
     import c.universe._
     val lhs = Applicative.impl0[Task, T, Ctx](c)(reify(Result.Success(t.splice)).tree)
 
-    mill.moduledefs.Cacher.impl0[TargetImpl, T](c)(
+    mill.moduledefs.Cacher.impl0[TargetImpl[T]](c)(
       reify(
         new TargetImpl[T](
           lhs.splice,
@@ -96,7 +96,7 @@ object Target extends TargetGenerated with Applicative.Applyer[Task, Task, Resul
                                          cl: c.Expr[Caller[mill.define.Task.Module]],
                                          o: c.Expr[Overrides]): c.Expr[Target[T]] = {
     import c.universe._
-    mill.moduledefs.Cacher.impl0[Target, T](c)(
+    mill.moduledefs.Cacher.impl0[Target[T]](c)(
       reify(
         new TargetImpl[T](
           Applicative.impl0[Task, T, Ctx](c)(t.tree).splice,
@@ -127,7 +127,7 @@ object Target extends TargetGenerated with Applicative.Applyer[Task, Task, Resul
                                        cl: c.Expr[Caller[mill.define.Task.Module]],
                                        o: c.Expr[Overrides]): c.Expr[Target[T]] = {
     import c.universe._
-    mill.moduledefs.Cacher.impl0[Target, T](c)(
+    mill.moduledefs.Cacher.impl0[Target[T]](c)(
       reify(
         new TargetImpl[T](
           t.splice,
@@ -148,7 +148,37 @@ object Target extends TargetGenerated with Applicative.Applyer[Task, Task, Resul
                  cl: Caller[mill.define.Task.Module],
                  o: Overrides): Command[T] = macro commandImpl[T]
 
-  def source(path: ammonite.ops.Path) = new Source(path)
+  def source[T](value: Result[T])
+               (implicit r: R[T],
+                w: W[T],
+                e: sourcecode.Enclosing,
+                n: sourcecode.Name,
+                cl: Caller[mill.define.Task.Module],
+                o: Overrides): Source[T] = macro sourceImpl[T]
+
+  def sourceImpl[T: c.WeakTypeTag](c: Context)
+                                  (value: c.Expr[T])
+                                  (r: c.Expr[R[T]],
+                                   w: c.Expr[W[T]],
+                                   e: c.Expr[sourcecode.Enclosing],
+                                   n: c.Expr[sourcecode.Name],
+                                   cl: c.Expr[Caller[mill.define.Task.Module]],
+                                   o: c.Expr[Overrides]): c.Expr[Source[T]] = {
+    import c.universe._
+
+    mill.moduledefs.Cacher.impl0[Source[T]](c)(
+      reify(
+        new Source[T](
+          Applicative.impl[Task, T, Ctx](c)(value).splice,
+          e.splice.value,
+          cl.splice.value,
+          n.splice.value,
+          upickle.default.ReadWriter(w.splice.write, r.splice.read),
+          o.splice.value
+        )
+      )
+    )
+  }
 
   def command[T](t: Task[T])
                 (implicit c: Caller[Task.Module],
@@ -197,7 +227,7 @@ object Target extends TargetGenerated with Applicative.Applyer[Task, Task, Resul
     import c.universe._
 
 
-    mill.moduledefs.Cacher.impl0[Persistent, T](c)(
+    mill.moduledefs.Cacher.impl0[Persistent[T]](c)(
       reify(
         new Persistent[T](
           Applicative.impl[Task, T, Ctx](c)(t).splice,
@@ -264,14 +294,15 @@ class Persistent[+T](t: Task[T],
   override def flushDest = false
   override def asPersistent = Some(this)
 }
-object Source{
-  implicit def apply(p: ammonite.ops.Path) = new Source(p)
-}
-class Source(path: ammonite.ops.Path) extends Task[PathRef]{
-  def handle = PathRef(path)
-  def evaluate(args: Ctx) = handle
-  override def sideHash = handle.hashCode()
-  val inputs = Nil
+class Source[T](t: Task[T],
+                val enclosing: String,
+                val owner: Task.Module,
+                val name: String,
+                val readWrite: RW[_],
+                val overrides: Int) extends Target[T]{
+  val inputs = Seq(t)
+  def evaluate(args: Ctx) = args[T](0)
+  override def sideHash = util.Random.nextInt()
 }
 
 object Task {
@@ -280,9 +311,7 @@ object Task {
   trait TaskModule extends Module {
     def defaultCommandName(): String
   }
-  trait Module extends mill.moduledefs.Cacher[Target]{
-    def wrapCached[T](t: Target[T], enclosing: String): Target[T] = t
-  }
+  trait Module extends mill.moduledefs.Cacher
 
   class Task0[T](t: T) extends Task[T]{
     lazy val t0 = t
diff --git a/core/src/main/scala/mill/eval/Evaluator.scala b/core/src/main/scala/mill/eval/Evaluator.scala
index 6fc0661a..4d94d3d9 100644
--- a/core/src/main/scala/mill/eval/Evaluator.scala
+++ b/core/src/main/scala/mill/eval/Evaluator.scala
@@ -24,6 +24,7 @@ case class Labelled[T](target: NamedTask[T],
   }
 }
 class Evaluator[T](val workspacePath: Path,
+                   val basePath: Path,
                    val mapping: Discovered.Mapping[T],
                    log: Logger,
                    val classLoaderSig: Seq[(Path, Long)] = Evaluator.classLoaderSig){
@@ -69,7 +70,13 @@ class Evaluator[T](val workspacePath: Path,
     for((k, vs) <- sortedGroups.items()){
       failing.addAll(k, vs.items.flatMap(results.get).collect{case f: Result.Failing => f})
     }
-    Evaluator.Results(goals.indexed.map(results), evaluated, transitive, failing)
+    Evaluator.Results(
+      goals.indexed.map(results),
+      evaluated,
+      transitive,
+      failing,
+      results
+    )
   }
 
 
@@ -87,10 +94,11 @@ class Evaluator[T](val workspacePath: Path,
 
     terminal match{
       case Left(task) =>
-        evaluateGroup(group, results, paths = None, maybeTargetLabel = None)
+        evaluateGroup(group, results, groupBasePath = None, paths = None, maybeTargetLabel = None)
       case Right(labelledTarget) =>
-        val paths = Evaluator.resolveDestPaths(workspacePath, labelledTarget)
-        mkdir(paths.base)
+        val paths = Evaluator.resolveDestPaths(workspacePath, labelledTarget.segments)
+        val groupBasePath = basePath / Evaluator.makeSegmentStrings(labelledTarget.segments)
+        mkdir(paths.out)
         val cached = for{
           json <- scala.util.Try(upickle.json.read(read(paths.meta))).toOption
           (cachedHash, terminalResult) <- scala.util.Try(upickle.default.readJs[(Int, upickle.Js.Value)](json)).toOption
@@ -117,8 +125,10 @@ class Evaluator[T](val workspacePath: Path,
             val (newResults, newEvaluated) = evaluateGroup(
               group,
               results,
-              Some(paths),
-              maybeTargetLabel = Some(msgParts.mkString))
+              groupBasePath = Some(groupBasePath),
+              paths = Some(paths),
+              maybeTargetLabel = Some(msgParts.mkString)
+            )
 
             newResults(labelledTarget.target) match{
               case Result.Success(v) =>
@@ -148,6 +158,7 @@ class Evaluator[T](val workspacePath: Path,
 
   def evaluateGroup(group: OSet[Task[_]],
                     results: collection.Map[Task[_], Result[Any]],
+                    groupBasePath: Option[Path],
                     paths: Option[Evaluator.Paths],
                     maybeTargetLabel: Option[String]) = {
 
@@ -183,6 +194,7 @@ class Evaluator[T](val workspacePath: Path,
           val args = new Ctx(
             targetInputValues.toArray[Any],
             paths.map(_.dest).orNull,
+            groupBasePath.orNull,
             multiLogger,
             new Ctx.LoaderCtx{
               def load[T](x: Ctx.Loader[T]): T = {
@@ -225,15 +237,16 @@ class Evaluator[T](val workspacePath: Path,
 
 
 object Evaluator{
-  case class Paths(base: Path, dest: Path, meta: Path, log: Path)
-  def resolveDestPaths(workspacePath: Path, t: Labelled[_]): Paths = {
-    resolveDestPaths(workspacePath, t.segments)
+  case class Paths(out: Path,
+                   dest: Path,
+                   meta: Path,
+                   log: Path)
+  def makeSegmentStrings(segments: Seq[Segment]) = segments.flatMap{
+    case Mirror.Segment.Label(s) => Seq(s)
+    case Mirror.Segment.Cross(values) => values.map(_.toString)
   }
   def resolveDestPaths(workspacePath: Path, segments: Seq[Segment]): Paths = {
-    val segmentStrings = segments.flatMap{
-      case Mirror.Segment.Label(s) => Seq(s)
-      case Mirror.Segment.Cross(values) => values.map(_.toString)
-    }
+    val segmentStrings = makeSegmentStrings(segments)
     val targetPath = workspacePath / segmentStrings
     Paths(targetPath, targetPath / 'dest, targetPath / "meta.json", targetPath / 'log)
   }
@@ -248,7 +261,8 @@ object Evaluator{
   case class Results(rawValues: Seq[Result[Any]],
                      evaluated: OSet[Task[_]],
                      transitive: OSet[Task[_]],
-                     failing: MultiBiMap[Either[Task[_], Labelled[_]], Result.Failing]){
+                     failing: MultiBiMap[Either[Task[_], Labelled[_]], Result.Failing],
+                     results: collection.Map[Task[_], Result[Any]]){
     def values = rawValues.collect{case Result.Success(v) => v}
   }
 }
diff --git a/core/src/main/scala/mill/main/MainRunner.scala b/core/src/main/scala/mill/main/MainRunner.scala
index f354e838..8f58f114 100644
--- a/core/src/main/scala/mill/main/MainRunner.scala
+++ b/core/src/main/scala/mill/main/MainRunner.scala
@@ -1,9 +1,9 @@
 package mill.main
-import ammonite.interp.Preprocessor
+import ammonite.interp.{Interpreter, Preprocessor}
 import ammonite.ops.Path
 import ammonite.util.{Imports, Name, Res, Util}
 import mill.discover.Discovered
-import mill.eval.Evaluator
+import mill.eval.{Evaluator, PathRef}
 import upickle.Js
 
 /**
@@ -33,9 +33,20 @@ class MainRunner(config: ammonite.main.Cli.Config,
             )
             result match{
               case Res.Success(data) =>
-                val (eval, evaluationWatches, res) = data
+                val (eval, evaluationWatches0, res) = data
+                val alreadyStale = evaluationWatches0.exists(p => p.sig != PathRef(p.path, p.quick).sig)
+                // If the file changed between the creation of the original
+                // `PathRef` and the current moment, use random junk .sig values
+                // to force an immediate re-run. Otherwise calculate the
+                // pathSignatures the same way Ammonite would and hand over the
+                // values, so Ammonite can watch them and only re-run if they
+                // subsequently change
+                val evaluationWatches =
+                  if (alreadyStale) evaluationWatches0.map(_.path -> util.Random.nextLong())
+                  else evaluationWatches0.map(p => p.path -> Interpreter.pathSignature(p.path))
+
                 lastEvaluator = Some((interpWatched, eval))
-                (Res.Success(res.flatMap(_._2)), interpWatched ++ evaluationWatches)
+                (Res(res.map(_.flatMap(_._2))), interpWatched ++ evaluationWatches)
               case _ =>
                 (result, interpWatched)
             }
diff --git a/core/src/main/scala/mill/main/ReplApplyHandler.scala b/core/src/main/scala/mill/main/ReplApplyHandler.scala
index d0c8365a..3832da68 100644
--- a/core/src/main/scala/mill/main/ReplApplyHandler.scala
+++ b/core/src/main/scala/mill/main/ReplApplyHandler.scala
@@ -11,6 +11,7 @@ object ReplApplyHandler{
     new ReplApplyHandler(
       new mill.eval.Evaluator(
         ammonite.ops.pwd / 'out,
+        ammonite.ops.pwd,
         mapping,
         new mill.util.PrintLogger(true)
       )
diff --git a/core/src/main/scala/mill/main/RunScript.scala b/core/src/main/scala/mill/main/RunScript.scala
index ad813026..dc7b74d6 100644
--- a/core/src/main/scala/mill/main/RunScript.scala
+++ b/core/src/main/scala/mill/main/RunScript.scala
@@ -6,7 +6,7 @@ import ammonite.interp.Interpreter
 import ammonite.ops.{Path, pwd, read}
 import ammonite.util.Util.CodeSource
 import ammonite.util.{Name, Res, Util}
-import mill.define
+import mill.{PathRef, define}
 import mill.define.Task
 import mill.discover.Mirror.Segment
 import mill.discover.{Discovered, Mirror}
@@ -27,8 +27,7 @@ object RunScript{
                 path: Path,
                 interp: ammonite.interp.Interpreter,
                 scriptArgs: Seq[String],
-                lastEvaluator: Option[(Seq[(Path, Long)], Evaluator[_])])
-  : Res[(Evaluator[_], Seq[(Path, Long)], Seq[(Any, Option[Js.Value])])] = {
+                lastEvaluator: Option[(Seq[(Path, Long)], Evaluator[_])]) = {
 
     val log = new PrintLogger(true)
     for{
@@ -40,15 +39,10 @@ object RunScript{
         case _ =>
           interp.watch(path)
           for(mapping <- evaluateMapping(wd, path, interp))
-          yield new Evaluator(pwd / 'out, mapping, log)
+          yield new Evaluator(pwd / 'out, pwd, mapping, log)
       }
-      evaluationWatches = mutable.Buffer.empty[(Path, Long)]
-      res <- Res(evaluateTarget(
-        evaluator,
-        scriptArgs,
-        p => evaluationWatches.append((p, Interpreter.pathSignature(p)))
-      ))
-    } yield (evaluator, evaluationWatches, res)
+      (watches, res) <- Res(evaluateTarget(evaluator, scriptArgs))
+    } yield (evaluator, watches, res)
   }
 
   def watchedSigUnchanged(sig: Seq[(Path, Long)]) = {
@@ -98,8 +92,7 @@ object RunScript{
     } yield mapping
   }
   def evaluateTarget[T](evaluator: Evaluator[_],
-                        scriptArgs: Seq[String],
-                        watch: Path => Unit) = {
+                        scriptArgs: Seq[String]) = {
 
     val Seq(selectorString, rest @_*) = scriptArgs
     for {
@@ -112,17 +105,18 @@ object RunScript{
         sel, evaluator.mapping.mirror, evaluator.mapping.base,
         rest, crossSelectors, Nil
       )
-      res <- evaluate(evaluator, target, watch)
-    } yield res
+      (watched, res) = evaluate(evaluator, target)
+    } yield (watched, res)
   }
   def evaluate(evaluator: Evaluator[_],
-               target: Task[Any],
-               watch: Path => Unit): Either[String, Seq[(Any, Option[upickle.Js.Value])]] = {
+               target: Task[Any]): (Seq[PathRef], Either[String, Seq[(Any, Option[upickle.Js.Value])]]) = {
     val evaluated = evaluator.evaluate(OSet(target))
-    evaluated.transitive.foreach {
-      case t: define.Source => watch(t.handle.path)
-      case _ => // do nothing
-    }
+    val watched = evaluated.results
+      .iterator
+      .collect {
+        case (t: define.Source[_], Result.Success(p: PathRef)) => p
+      }
+      .toSeq
 
     val errorStr =
       (for((k, fs) <- evaluated.failing.items()) yield {
@@ -151,8 +145,8 @@ object RunScript{
           }
         }
 
-        Right(evaluated.values.zip(json))
-      case n => Left(s"$n targets failed\n$errorStr")
+        watched -> Right(evaluated.values.zip(json))
+      case n => watched -> Left(s"$n targets failed\n$errorStr")
     }
   }
 
diff --git a/core/src/main/scala/mill/util/Ctx.scala b/core/src/main/scala/mill/util/Ctx.scala
index 9604021a..60e6bdbf 100644
--- a/core/src/main/scala/mill/util/Ctx.scala
+++ b/core/src/main/scala/mill/util/Ctx.scala
@@ -2,7 +2,7 @@ package mill.util
 
 import ammonite.ops.Path
 import mill.define.Applicative.ImplicitStub
-import mill.util.Ctx.{ArgCtx, DestCtx, LoaderCtx, LogCtx}
+import mill.util.Ctx.{ArgCtx, BaseCtx, DestCtx, LoaderCtx, LogCtx}
 
 import scala.annotation.compileTimeOnly
 import scala.language.implicitConversions
@@ -13,15 +13,23 @@ object Ctx{
   implicit def taskCtx: Ctx = ???
 
   object DestCtx {
-    implicit def pathToCtx(path: Path): DestCtx =
-      new DestCtx { def dest: Path = path }
+    implicit def pathToCtx(path: Path): DestCtx = new DestCtx { def dest = path }
   }
   trait DestCtx{
     def dest: Path
   }
+  trait BaseCtx{
+    def base: Path
+  }
+  object BaseCtx {
+    implicit def pathToCtx(path: Path): BaseCtx = new BaseCtx { def base = path }
+  }
   trait LogCtx{
     def log: Logger
   }
+  object LogCtx{
+    implicit def logToCtx(l: Logger): LogCtx = new LogCtx { def log = l }
+  }
   trait ArgCtx{
     def args: IndexedSeq[_]
   }
@@ -34,12 +42,14 @@ object Ctx{
 }
 class Ctx(val args: IndexedSeq[_],
           val dest: Path,
+          val base: Path,
           val log: Logger,
           workerCtx0: Ctx.LoaderCtx)
   extends DestCtx
   with LogCtx
   with ArgCtx
-  with LoaderCtx{
+  with LoaderCtx
+  with BaseCtx{
 
   def load[T](x: Ctx.Loader[T]): T = workerCtx0.load(x)
   def length = args.length