Add a `visualize` command to render portions of the build graph as SVG (#349)

* Transitive reduction of visualized graph via jgrapht now works

```
out/dev/launcher/dest/run -i visualize __.compile _
out/dev/launcher/dest/run -i visualize core.__
```

* Move test running logic from scalaworker into scalalib

This is to try and reduce the size of the classpath we are passing to the test runner subprocess, in an attempt to fix the command-line-too-long errors we're getting in Appveyor. Now the test runner subprocess should no longer need Zinc or all of it's transitive dependencies

* - Break out `GraphvizTools` into a separate Mill module, to avoid bloating the main jar and try to shorten the `MILL_SCALA_WORKER` classpath being sent to scalajslib.test (which is blowing up on windows as the CLI command is too long)

- Move the meat of `resolveDependencies` from `scalalib` to `main`, to support resolving mill modules

- DRY up resolution of mill modules in `Util.millProjectModule`

* fix mill module resolution

* Tweaks to try and make zinc work again...

* Tweak `Module#reflect` to try and make it happy with `visualize` module...

* fix integration test classpath

* move visualization into it's own module

16 files changed, 465 insertions, 256 deletions

diff --git a/build.sc b/build.sc
index ac6c2ea9..c3dd8fa8 100755
--- a/build.sc
+++ b/build.sc
@@ -9,6 +9,7 @@ import mill._
 import mill.scalalib._
 import publish._
 import mill.modules.Jvm.createAssembly
+
 import upickle.Js
 trait MillPublishModule extends PublishModule{
 
@@ -124,6 +125,18 @@ object main extends MillModule {
       def ivyDeps = Agg(ivy"com.novocode:junit-interface:0.11")
     }
   }
+
+  object graphviz extends MillModule{
+    def moduleDeps = Seq(main, scalalib)
+
+    def ivyDeps = Agg(
+      ivy"guru.nidi:graphviz-java:0.2.3",
+      ivy"org.jgrapht:jgrapht-core:1.2.0"
+    )
+    def testArgs = Seq(
+      "-DMILL_GRAPHVIZ=" + runClasspath().map(_.path).mkString(",")
+    )
+  }
 }
 
 
@@ -148,7 +161,12 @@ object scalalib extends MillModule {
       genTask(scalajslib)()
 
     worker.testArgs() ++
-    Seq("-Djna.nosys=true") ++ Seq("-DMILL_BUILD_LIBRARIES=" + genIdeaArgs.map(_.path).mkString(","))
+    main.graphviz.testArgs() ++
+    Seq(
+      "-Djna.nosys=true",
+      "-DMILL_BUILD_LIBRARIES=" + genIdeaArgs.map(_.path).mkString(","),
+      "-DMILL_SCALA_LIB=" + runClasspath().map(_.path).mkString(",")
+    )
   }
 
   object worker extends MillModule{
@@ -229,6 +247,7 @@ object integration extends MillModule{
     Seq(
       "-DMILL_TESTNG=" + testng.runClasspath().map(_.path).mkString(","),
       "-DMILL_VERSION=" + build.publishVersion()._2,
+      "-DMILL_SCALA_LIB=" + scalalib.runClasspath().map(_.path).mkString(","),
       "-Djna.nosys=true"
     ) ++
     (for((k, v) <- testRepos()) yield s"-D$k=$v")
diff --git a/core/src/mill/define/Module.scala b/core/src/mill/define/Module.scala
index 6abcc75a..3f91b524 100644
--- a/core/src/mill/define/Module.scala
+++ b/core/src/mill/define/Module.scala
@@ -64,9 +64,9 @@ object Module{
           !m.getName.contains('$') &&
           m.getParameterCount == 0 &&
           (m.getModifiers & Modifier.STATIC) == 0 &&
+          (m.getModifiers & Modifier.ABSTRACT) == 0 &&
           runtimeCls.isAssignableFrom(m.getReturnType)
       } yield m.invoke(outer).asInstanceOf[T]
-
     }
     def reflectNames[T: ClassTag] = {
       val runtimeCls = implicitly[ClassTag[T]].runtimeClass
diff --git a/main/graphviz/src/mill/main/graphviz/GraphvizTools.scala b/main/graphviz/src/mill/main/graphviz/GraphvizTools.scala
new file mode 100644
index 00000000..300bd98a
--- /dev/null
+++ b/main/graphviz/src/mill/main/graphviz/GraphvizTools.scala
@@ -0,0 +1,66 @@
+package mill.main.graphviz
+import ammonite.ops.Path
+import mill.define.{Graph, NamedTask}
+import org.jgrapht.graph.{DefaultEdge, SimpleDirectedGraph}
+object GraphvizTools{
+  def apply(rs: Seq[NamedTask[Any]], dest: Path) = {
+    val transitive = Graph.transitiveTargets(rs.distinct)
+    val topoSorted = Graph.topoSorted(transitive)
+    val goalSet = rs.toSet
+    val sortedGroups = Graph.groupAroundImportantTargets(topoSorted){
+      case x: NamedTask[Any] if goalSet.contains(x) => x
+    }
+    import guru.nidi.graphviz.model.Factory._
+    import guru.nidi.graphviz.engine.{Format, Graphviz}
+
+    val edgesIterator =
+      for((k, vs) <- sortedGroups.items())
+        yield (
+          k,
+          for {
+            v <- vs.items
+            dest <- v.inputs.collect { case v: NamedTask[Any] => v }
+            if goalSet.contains(dest)
+          } yield dest
+        )
+
+    val edges = edgesIterator.map{case (k, v) => (k, v.toArray.distinct)}.toArray
+
+    val indexToTask = edges.flatMap{case (k, vs) => Iterator(k) ++ vs}.distinct
+    val taskToIndex = indexToTask.zipWithIndex.toMap
+
+    val jgraph = new SimpleDirectedGraph[Int, DefaultEdge](classOf[DefaultEdge])
+
+    for(i <- indexToTask.indices) jgraph.addVertex(i)
+    for((src, dests) <- edges; dest <- dests) {
+      jgraph.addEdge(taskToIndex(src), taskToIndex(dest))
+    }
+
+
+    org.jgrapht.alg.TransitiveReduction.INSTANCE.reduce(jgraph)
+    val nodes = indexToTask.map(t => node(t.ctx.segments.render))
+
+    var g = graph("example1").directed
+    for(i <- indexToTask.indices){
+      val outgoing = for{
+        e <- edges(i)._2
+        j = taskToIndex(e)
+        if jgraph.containsEdge(i, j)
+      } yield nodes(j)
+      g = g.`with`(nodes(i).link(outgoing:_*))
+    }
+
+    val gv = Graphviz.fromGraph(g).totalMemory(100 * 1000 * 1000)
+    val outputs = Seq(
+      Format.PLAIN -> "out.txt",
+      Format.XDOT -> "out.dot",
+      Format.JSON -> "out.json",
+      Format.PNG -> "out.png",
+      Format.SVG -> "out.svg"
+    )
+    for((fmt, name) <- outputs) {
+      gv.render(fmt).toFile((dest / name).toIO)
+    }
+    outputs.map(x => mill.PathRef(dest / x._2))
+  }
+}
\ No newline at end of file
diff --git a/main/src/mill/main/MainModule.scala b/main/src/mill/main/MainModule.scala
index 4fed5ec5..c7efbd21 100644
--- a/main/src/mill/main/MainModule.scala
+++ b/main/src/mill/main/MainModule.scala
@@ -1,12 +1,15 @@
 package mill.main
 
 import ammonite.ops.Path
-import mill.define.{NamedTask, Task}
-import mill.eval.{Evaluator, Result}
-import mill.util.{PrintLogger, Watched}
+import coursier.Cache
+import coursier.maven.MavenRepository
+import mill.T
+import mill.define.{Graph, NamedTask, Task}
+import mill.eval.{Evaluator, PathRef, Result}
+import mill.util.{Loose, PrintLogger, Watched}
 import pprint.{Renderer, Truncated}
 import upickle.Js
-
+import mill.util.JsonFormatters._
 object MainModule{
   def resolveTasks[T](evaluator: Evaluator[Any], targets: Seq[String], multiSelect: Boolean)
                      (f: List[NamedTask[Any]] => T) = {
@@ -217,4 +220,12 @@ trait MainModule extends mill.Module{
     }
   }
 
+  val visualize: VisualizeModule = new VisualizeModule {
+    def repositories = Seq(
+      Cache.ivy2Local,
+      MavenRepository("https://repo1.maven.org/maven2"),
+      MavenRepository("https://oss.sonatype.org/content/repositories/releases")
+    )
+  }
+
 }
diff --git a/main/src/mill/main/Resolve.scala b/main/src/mill/main/Resolve.scala
index 4b0ae27f..4baac312 100644
--- a/main/src/mill/main/Resolve.scala
+++ b/main/src/mill/main/Resolve.scala
@@ -188,28 +188,37 @@ object ResolveTasks extends Resolve[NamedTask[Any]]{
                       revSelectorsSoFar: List[Segment],
                       last: String,
                       discover: Discover[_],
-                      rest: Seq[String]) = {
-    val target =
-      obj
-        .millInternal
-        .reflect[Target[_]]
-        .find(_.label == last)
-        .map(Right(_))
-
-    val command = Resolve.invokeCommand(obj, last, discover, rest).headOption
-
-    command orElse target orElse Resolve.runDefault(obj, Segment.Label(last), discover, rest).flatten.headOption match {
-      case None =>
-        Resolve.errorMsgLabel(
-          singleModuleMeta(obj, discover, revSelectorsSoFar.isEmpty),
-          last,
-          revSelectorsSoFar
-        )
-
-      // Contents of `either` *must* be a `Task`, because we only select
-      // methods returning `Task` in the discovery process
-      case Some(either) => either.right.map(Seq(_))
-    }
+                      rest: Seq[String]) = last match{
+    case "__" =>
+      Right(
+        obj.millInternal.modules
+          .filter(_ != obj)
+          .flatMap(m => m.millInternal.reflect[Target[_]])
+      )
+    case "_" => Right(obj.millInternal.reflect[Target[_]])
+
+    case _ =>
+      val target =
+        obj
+          .millInternal
+          .reflect[Target[_]]
+          .find(_.label == last)
+          .map(Right(_))
+
+      val command = Resolve.invokeCommand(obj, last, discover, rest).headOption
+
+      command orElse target orElse Resolve.runDefault(obj, Segment.Label(last), discover, rest).flatten.headOption match {
+        case None =>
+          Resolve.errorMsgLabel(
+            singleModuleMeta(obj, discover, revSelectorsSoFar.isEmpty),
+            last,
+            revSelectorsSoFar
+          )
+
+        // Contents of `either` *must* be a `Task`, because we only select
+        // methods returning `Task` in the discovery process
+        case Some(either) => either.right.map(Seq(_))
+      }
   }
 }
 object Resolve{
diff --git a/main/src/mill/main/VisualizeModule.scala b/main/src/mill/main/VisualizeModule.scala
new file mode 100644
index 00000000..00651a8e
--- /dev/null
+++ b/main/src/mill/main/VisualizeModule.scala
@@ -0,0 +1,37 @@
+package mill.main
+
+import ammonite.ops.Path
+import coursier.core.Repository
+import mill.T
+import mill.eval.{Evaluator, PathRef, Result}
+
+trait VisualizeModule extends mill.define.TaskModule{
+  def repositories: Seq[Repository]
+  def defaultCommandName() = "run"
+  def classpath = T{
+    mill.modules.Util.millProjectModule("MILL_GRAPHVIZ", "mill-main-graphviz", repositories)
+  }
+  /**
+    * Given a set of tasks, prints out the execution plan of what tasks will be
+    * executed in what order, without actually executing them.
+    */
+  def run(evaluator: Evaluator[Any], targets: String*) = mill.T.command{
+    val resolved = RunScript.resolveTasks(
+      mill.main.ResolveTasks, evaluator, targets, multiSelect = true
+    )
+    resolved match{
+      case Left(err) => Result.Failure(err)
+      case Right(rs) =>
+        Result.Success(
+          mill.modules.Jvm.inprocess(classpath().map(_.path), false, isolated = false, cl => {
+            cl.loadClass("mill.main.graphviz.GraphvizTools")
+              .getMethod("apply", classOf[Seq[_]], classOf[Path])
+              .invoke(null, rs, T.ctx().dest)
+              .asInstanceOf[Seq[PathRef]]
+          })
+        )
+
+    }
+  }
+
+}
diff --git a/main/src/mill/modules/Jvm.scala b/main/src/mill/modules/Jvm.scala
index d32d3612..1a28189f 100644
--- a/main/src/mill/modules/Jvm.scala
+++ b/main/src/mill/modules/Jvm.scala
@@ -8,9 +8,11 @@ import java.nio.file.attribute.PosixFilePermission
 import java.util.jar.{JarEntry, JarFile, JarOutputStream}
 
 import ammonite.ops._
+import ammonite.util.Util
+import coursier.{Cache, Dependency, Fetch, Repository, Resolution}
 import geny.Generator
 import mill.main.client.InputPumper
-import mill.eval.PathRef
+import mill.eval.{PathRef, Result}
 import mill.util.{Ctx, IO}
 import mill.util.Loose.Agg
 
@@ -78,7 +80,7 @@ object Jvm {
                classPath: Agg[Path],
                mainArgs: Seq[String] = Seq.empty)
               (implicit ctx: Ctx): Unit = {
-    inprocess(classPath, classLoaderOverrideSbtTesting = false, cl => {
+    inprocess(classPath, classLoaderOverrideSbtTesting = false, isolated = true, cl => {
       getMainMethod(mainClass, cl).invoke(null, mainArgs.toArray)
     })
   }
@@ -101,6 +103,7 @@ object Jvm {
 
   def inprocess[T](classPath: Agg[Path],
                    classLoaderOverrideSbtTesting: Boolean,
+                   isolated: Boolean,
                    body: ClassLoader => T)
                   (implicit ctx: Ctx.Home): T = {
     val urls = classPath.map(_.toIO.toURI.toURL)
@@ -111,8 +114,11 @@ object Jvm {
           Some(outerClassLoader.loadClass(name))
         else None
       })
-    } else {
+    } else if (isolated){
+
       mill.util.ClassLoader.create(urls.toVector, null)
+    }else{
+      mill.util.ClassLoader.create(urls.toVector, getClass.getClassLoader)
     }
     val oldCl = Thread.currentThread().getContextClassLoader
     Thread.currentThread().setContextClassLoader(cl)
@@ -376,4 +382,87 @@ object Jvm {
     PathRef(outputPath)
   }
 
+  /**
+    * Resolve dependencies using Coursier.
+    *
+    * We do not bother breaking this out into the separate ScalaWorker classpath,
+    * because Coursier is already bundled with mill/Ammonite to support the
+    * `import $ivy` syntax.
+    */
+  def resolveDependencies(repositories: Seq[Repository],
+                          deps: TraversableOnce[coursier.Dependency],
+                          force: TraversableOnce[coursier.Dependency],
+                          sources: Boolean = false,
+                          mapDependencies: Option[Dependency => Dependency] = None): Result[Agg[PathRef]] = {
+
+    val (_, resolution) = resolveDependenciesMetadata(
+      repositories, deps, force, mapDependencies
+    )
+    val errs = resolution.metadataErrors
+    if(errs.nonEmpty) {
+      val header =
+        s"""|
+            |Resolution failed for ${errs.length} modules:
+            |--------------------------------------------
+            |""".stripMargin
+
+      val errLines = errs.map {
+        case ((module, vsn), errMsgs) => s"  ${module.trim}:$vsn \n\t" + errMsgs.mkString("\n\t")
+      }.mkString("\n")
+      val msg = header + errLines + "\n"
+      Result.Failure(msg)
+    } else {
+
+      def load(artifacts: Seq[coursier.Artifact]) = {
+        val logger = None
+        val loadedArtifacts = scalaz.concurrent.Task.gatherUnordered(
+          for (a <- artifacts)
+            yield coursier.Cache.file(a, logger = logger).run
+              .map(a.isOptional -> _)
+        ).unsafePerformSync
+
+        val errors = loadedArtifacts.collect {
+          case (false, scalaz.-\/(x)) => x
+          case (true, scalaz.-\/(x)) if !x.notFound => x
+        }
+        val successes = loadedArtifacts.collect { case (_, scalaz.\/-(x)) => x }
+        (errors, successes)
+      }
+
+      val sourceOrJar =
+        if (sources) resolution.classifiersArtifacts(Seq("sources"))
+        else resolution.artifacts(true)
+      val (errors, successes) = load(sourceOrJar)
+      if(errors.isEmpty){
+        mill.Agg.from(
+          successes.map(p => PathRef(Path(p), quick = true)).filter(_.path.ext == "jar")
+        )
+      }else{
+        val errorDetails = errors.map(e => s"${ammonite.util.Util.newLine}  ${e.describe}").mkString
+        Result.Failure("Failed to load source dependencies" + errorDetails)
+      }
+    }
+  }
+
+
+  def resolveDependenciesMetadata(repositories: Seq[Repository],
+                                  deps: TraversableOnce[coursier.Dependency],
+                                  force: TraversableOnce[coursier.Dependency],
+                                  mapDependencies: Option[Dependency => Dependency] = None) = {
+
+    val forceVersions = force
+      .map(mapDependencies.getOrElse(identity[Dependency](_)))
+      .map{d => d.module -> d.version}
+      .toMap
+
+    val start = Resolution(
+      deps.map(mapDependencies.getOrElse(identity[Dependency](_))).toSet,
+      forceVersions = forceVersions,
+      mapDependencies = mapDependencies
+    )
+
+    val fetch = Fetch.from(repositories, Cache.fetch())
+    val resolution = start.process.run(fetch).unsafePerformSync
+    (deps.toSeq, resolution)
+  }
 }
diff --git a/main/src/mill/modules/Util.scala b/main/src/mill/modules/Util.scala
index 3029411c..da7407d0 100644
--- a/main/src/mill/modules/Util.scala
+++ b/main/src/mill/modules/Util.scala
@@ -2,8 +2,9 @@ package mill.modules
 
 
 import ammonite.ops.{Path, RelPath, empty, mkdir, read}
+import coursier.Repository
 import mill.eval.PathRef
-import mill.util.{Ctx, IO}
+import mill.util.{Ctx, IO, Loose}
 
 object Util {
   def download(url: String, dest: RelPath = "download")(implicit ctx: Ctx.Dest) = {
@@ -54,4 +55,27 @@ object Util {
     })()
     PathRef(ctx.dest / dest)
   }
+
+  def millProjectModule(key: String,
+                        artifact: String,
+                        repositories: Seq[Repository],
+                        resolveFilter: Path => Boolean = _ => true) = {
+    val localPath = sys.props(key)
+    if (localPath != null) {
+      mill.eval.Result.Success(
+        Loose.Agg.from(localPath.split(',').map(p => PathRef(Path(p), quick = true)))
+      )
+    } else {
+      mill.modules.Jvm.resolveDependencies(
+        repositories,
+        Seq(
+          coursier.Dependency(
+            coursier.Module("com.lihaoyi", artifact + "_2.12"),
+            sys.props("MILL_VERSION")
+          )
+        ),
+        Nil
+      ).map(_.filter(x => resolveFilter(x.path)))
+    }
+  }
 }
diff --git a/main/test/src/mill/util/TestEvaluator.scala b/main/test/src/mill/util/TestEvaluator.scala
index 26e4b4b5..bcfcc9bf 100644
--- a/main/test/src/mill/util/TestEvaluator.scala
+++ b/main/test/src/mill/util/TestEvaluator.scala
@@ -24,11 +24,11 @@ class TestEvaluator[T <: TestUtil.BaseModule](module: T)
                                              tp: TestPath){
   val outPath =  TestUtil.getOutPath()
 
-  val logger = DummyLogger
-//  val logger = new PrintLogger(
-//    true,
-//    ammonite.util.Colors.Default, System.out, System.out, System.err, System.in
-// )
+//  val logger = DummyLogger
+  val logger = new PrintLogger(
+    true,
+    ammonite.util.Colors.Default, System.out, System.out, System.err, System.in
+ )
   val evaluator = new Evaluator(Ctx.defaultHome, outPath, TestEvaluator.externalOutPath, module, logger)
 
   def apply[T](t: Task[T]): Either[Result.Failing[T], (T, Int)] = {
diff --git a/scalajslib/src/mill/scalajslib/ScalaJSModule.scala b/scalajslib/src/mill/scalajslib/ScalaJSModule.scala
index 1596cdb6..1b99f142 100644
--- a/scalajslib/src/mill/scalajslib/ScalaJSModule.scala
+++ b/scalajslib/src/mill/scalajslib/ScalaJSModule.scala
@@ -7,7 +7,7 @@ import coursier.maven.MavenRepository
 import mill.eval.{PathRef, Result}
 import mill.eval.Result.Success
 import mill.scalalib.Lib.resolveDependencies
-import mill.scalalib.{DepSyntax, Lib, TestModule}
+import mill.scalalib.{DepSyntax, Lib, TestModule, TestRunner}
 import mill.util.{Ctx, Loose}
 
 trait ScalaJSModule extends scalalib.ScalaModule { outer =>
@@ -27,20 +27,12 @@ trait ScalaJSModule extends scalalib.ScalaModule { outer =>
 
   def sjsBridgeClasspath = T {
     val jsBridgeKey = "MILL_SCALAJS_BRIDGE_" + scalaJSBridgeVersion().replace('.', '_')
-    val jsBridgePath = sys.props(jsBridgeKey)
-    if (jsBridgePath != null) Success(
-      Agg(PathRef(Path(jsBridgePath), quick = true))
-    ) else resolveDependencies(
-      Seq(
-        Cache.ivy2Local,
-        MavenRepository("https://repo1.maven.org/maven2"),
-        MavenRepository("https://oss.sonatype.org/content/repositories/releases")
-      ),
-      Lib.depToDependency(_, "2.12.4", ""),
-      Seq(
-        ivy"com.lihaoyi::mill-scalajslib-jsbridges-${scalaJSBridgeVersion()}:${sys.props("MILL_VERSION")}"
-      )
-    ).map(_.filter(_.path.toString.contains("mill-scalajslib-jsbridges")))
+    mill.modules.Util.millProjectModule(
+      jsBridgeKey,
+      s"mill-scalajslib-jsbridges-${scalaJSBridgeVersion()}",
+      repositories,
+      resolveFilter = _.toString.contains("mill-scalajslib-jsbridges")
+    )
   }
 
   def scalaJSLinkerClasspath: T[Loose.Agg[PathRef]] = T{
@@ -190,7 +182,7 @@ trait TestScalaJSModule extends ScalaJSModule with TestModule {
         fastOptTest().path.toIO
       )
 
-    val (doneMsg, results) = Lib.runTests(
+    val (doneMsg, results) = TestRunner.runTests(
         _ => Seq(framework),
         runClasspath().map(_.path),
         Agg(compile().classes.path),
diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala
index a05e1778..26c58a8b 100644
--- a/scalalib/src/mill/scalalib/JavaModule.scala
+++ b/scalalib/src/mill/scalalib/JavaModule.scala
@@ -303,8 +303,8 @@ trait TestModule extends JavaModule with TaskModule {
     val outputPath = T.ctx().dest/"out.json"
 
     Jvm.subprocess(
-      mainClass = "mill.scalalib.worker.ScalaWorker",
-      classPath = ScalaWorkerModule.classpath(),
+      mainClass = "mill.scalalib.TestRunner",
+      classPath = ScalaWorkerModule.scalalibClasspath().map(_.path),
       jvmArgs = forkArgs(),
       envArgs = forkEnv(),
       mainArgs =
@@ -330,7 +330,7 @@ trait TestModule extends JavaModule with TaskModule {
   def testLocal(args: String*) = T.command{
     val outputPath = T.ctx().dest/"out.json"
 
-    Lib.runTests(
+    TestRunner.runTests(
       TestRunner.frameworks(testFrameworks()),
       runClasspath().map(_.path),
       Agg(compile().classes.path),
diff --git a/scalalib/src/mill/scalalib/Lib.scala b/scalalib/src/mill/scalalib/Lib.scala
index bda8e8f8..7bd0a5ff 100644
--- a/scalalib/src/mill/scalalib/Lib.scala
+++ b/scalalib/src/mill/scalalib/Lib.scala
@@ -109,23 +109,12 @@ object Lib{
                                   deps: TraversableOnce[Dep],
                                   mapDependencies: Option[Dependency => Dependency] = None) = {
     val depSeq = deps.toSeq
-    val flattened = depSeq.map(depToDependency)
-
-    val forceVersions = depSeq.filter(_.force)
-      .map(depToDependency)
-      .map(mapDependencies.getOrElse(identity[Dependency](_)))
-      .map{d => d.module -> d.version}
-      .toMap
-
-    val start = Resolution(
-      flattened.map(mapDependencies.getOrElse(identity[Dependency](_))).toSet,
-      forceVersions = forceVersions,
-      mapDependencies = mapDependencies
+    mill.modules.Jvm.resolveDependenciesMetadata(
+      repositories,
+      depSeq.map(depToDependency),
+      depSeq.filter(_.force).map(depToDependency),
+      mapDependencies
     )
-
-    val fetch = Fetch.from(repositories, Cache.fetch())
-    val resolution = start.process.run(fetch).unsafePerformSync
-    (flattened, resolution)
   }
   /**
     * Resolve dependencies using Coursier.
@@ -139,54 +128,14 @@ object Lib{
                           deps: TraversableOnce[Dep],
                           sources: Boolean = false,
                           mapDependencies: Option[Dependency => Dependency] = None): Result[Agg[PathRef]] = {
-
-    val (_, resolution) = resolveDependenciesMetadata(
-      repositories, depToDependency, deps, mapDependencies
+    val depSeq = deps.toSeq
+    mill.modules.Jvm.resolveDependencies(
+      repositories,
+      depSeq.map(depToDependency),
+      depSeq.filter(_.force).map(depToDependency),
+      sources,
+      mapDependencies
     )
-    val errs = resolution.metadataErrors
-    if(errs.nonEmpty) {
-      val header =
-        s"""|
-            |Resolution failed for ${errs.length} modules:
-            |--------------------------------------------
-            |""".stripMargin
-
-      val errLines = errs.map {
-        case ((module, vsn), errMsgs) => s"  ${module.trim}:$vsn \n\t" + errMsgs.mkString("\n\t")
-      }.mkString("\n")
-      val msg = header + errLines + "\n"
-      Result.Failure(msg)
-    } else {
-
-      def load(artifacts: Seq[coursier.Artifact]) = {
-        val logger = None
-        val loadedArtifacts = scalaz.concurrent.Task.gatherUnordered(
-          for (a <- artifacts)
-            yield coursier.Cache.file(a, logger = logger).run
-              .map(a.isOptional -> _)
-        ).unsafePerformSync
-
-        val errors = loadedArtifacts.collect {
-          case (false, scalaz.-\/(x)) => x
-          case (true, scalaz.-\/(x)) if !x.notFound => x
-        }
-        val successes = loadedArtifacts.collect { case (_, scalaz.\/-(x)) => x }
-        (errors, successes)
-      }
-
-      val sourceOrJar =
-        if (sources) resolution.classifiersArtifacts(Seq("sources"))
-        else resolution.artifacts(true)
-      val (errors, successes) = load(sourceOrJar)
-      if(errors.isEmpty){
-        Agg.from(
-          successes.map(p => PathRef(Path(p), quick = true)).filter(_.path.ext == "jar")
-        )
-      }else{
-        val errorDetails = errors.map(e => s"${Util.newLine}  ${e.describe}").mkString
-        Result.Failure("Failed to load source dependencies" + errorDetails)
-      }
-    }
   }
   def scalaCompilerIvyDeps(scalaVersion: String) = Agg[Dep](
     ivy"org.scala-lang:scala-compiler:$scalaVersion".forceVersion(),
@@ -202,75 +151,6 @@ object Lib{
       force = false
     )
 
-  def runTests(frameworkInstances: ClassLoader => Seq[sbt.testing.Framework],
-               entireClasspath: Agg[Path],
-               testClassfilePath: Agg[Path],
-               args: Seq[String])
-              (implicit ctx: Ctx.Log with Ctx.Home): (String, Seq[mill.scalalib.TestRunner.Result]) = {
-    Jvm.inprocess(entireClasspath, classLoaderOverrideSbtTesting = true, cl => {
-      val frameworks = frameworkInstances(cl)
-
-      val events = mutable.Buffer.empty[Event]
-
-      val doneMessages = frameworks.map { framework =>
-        val runner = framework.runner(args.toArray, args.toArray, cl)
-
-        val testClasses = discoverTests(cl, framework, testClassfilePath)
-
-        val tasks = runner.tasks(
-          for ((cls, fingerprint) <- testClasses.toArray)
-          yield new TaskDef(cls.getName.stripSuffix("$"), fingerprint, true, Array(new SuiteSelector))
-        )
-
-        val taskQueue = tasks.to[mutable.Queue]
-        while (taskQueue.nonEmpty){
-          val next = taskQueue.dequeue().execute(
-            new EventHandler {
-              def handle(event: Event) = events.append(event)
-            },
-            Array(
-              new Logger {
-                def debug(msg: String) = ctx.log.outputStream.println(msg)
-
-                def error(msg: String) = ctx.log.outputStream.println(msg)
-
-                def ansiCodesSupported() = true
-
-                def warn(msg: String) = ctx.log.outputStream.println(msg)
-
-                def trace(t: Throwable) = t.printStackTrace(ctx.log.outputStream)
-
-                def info(msg: String) = ctx.log.outputStream.println(msg)
-              })
-          )
-          taskQueue.enqueue(next:_*)
-        }
-        ctx.log.outputStream.println(runner.done())
-      }
-
-      val results = for(e <- events) yield {
-        val ex = if (e.throwable().isDefined) Some(e.throwable().get) else None
-        mill.scalalib.TestRunner.Result(
-          e.fullyQualifiedName(),
-          e.selector() match{
-            case s: NestedSuiteSelector => s.suiteId()
-            case s: NestedTestSelector => s.suiteId() + "." + s.testName()
-            case s: SuiteSelector => s.toString
-            case s: TestSelector => s.testName()
-            case s: TestWildcardSelector => s.testWildcard()
-          },
-          e.duration(),
-          e.status().toString,
-          ex.map(_.getClass.getName),
-          ex.map(_.getMessage),
-          ex.map(_.getStackTrace)
-        )
-      }
-
-      (doneMessages.mkString("\n"), results)
-    })
-  }
-
   def listClassFiles(base: Path): Iterator[String] = {
     if (base.isDir) ls.rec(base).toIterator.filter(_.ext == "class").map(_.relativeTo(base).toString)
     else {
diff --git a/scalalib/src/mill/scalalib/ScalaWorkerApi.scala b/scalalib/src/mill/scalalib/ScalaWorkerApi.scala
index a01c1c2c..6865c541 100644
--- a/scalalib/src/mill/scalalib/ScalaWorkerApi.scala
+++ b/scalalib/src/mill/scalalib/ScalaWorkerApi.scala
@@ -22,20 +22,16 @@ trait ScalaWorkerModule extends mill.Module{
   )
 
   def classpath = T{
-    val scalaWorkerJar = sys.props("MILL_SCALA_WORKER")
-    if (scalaWorkerJar != null) {
-      mill.eval.Result.Success(Loose.Agg.from(scalaWorkerJar.split(',').map(Path(_))))
-    } else {
-      resolveDependencies(
-        repositories,
-        Lib.depToDependency(_, "2.12.4", ""),
-        Seq(ivy"com.lihaoyi::mill-scalalib-worker:${sys.props("MILL_VERSION")}")
-      ).map(_.map(_.path))
-    }
+    mill.modules.Util.millProjectModule("MILL_SCALA_WORKER", "mill-scalalib-worker", repositories)
   }
+
+  def scalalibClasspath = T{
+    mill.modules.Util.millProjectModule("MILL_SCALA_LIB", "mill-scalalib", repositories)
+  }
+
   def worker: Worker[ScalaWorkerApi] = T.worker{
     val cl = mill.util.ClassLoader.create(
-      classpath().map(_.toNIO.toUri.toURL).toVector,
+      classpath().map(_.path.toNIO.toUri.toURL).toVector,
       getClass.getClassLoader
     )
     val cls = cl.loadClass("mill.scalalib.worker.ScalaWorker")
diff --git a/scalalib/src/mill/scalalib/TestRunner.scala b/scalalib/src/mill/scalalib/TestRunner.scala
index 7bee681f..a014df0c 100644
--- a/scalalib/src/mill/scalalib/TestRunner.scala
+++ b/scalalib/src/mill/scalalib/TestRunner.scala
@@ -1,7 +1,136 @@
 package mill.scalalib
+import ammonite.ops.Path
+import ammonite.util.Colors
+import mill.Agg
+import mill.modules.Jvm
+import mill.scalalib.Lib.discoverTests
+import mill.util.{Ctx, PrintLogger}
 import mill.util.JsonFormatters._
+import sbt.testing._
+
+import scala.collection.mutable
 object TestRunner {
 
+
+  def main(args: Array[String]): Unit = {
+    try{
+      var i = 0
+      def readArray() = {
+        val count = args(i).toInt
+        val slice = args.slice(i + 1, i + count + 1)
+        i = i + count + 1
+        slice
+      }
+      val frameworks = readArray()
+      val classpath = readArray()
+      val arguments = readArray()
+      val outputPath = args(i + 0)
+      val colored = args(i + 1)
+      val testCp = args(i + 2)
+      val homeStr = args(i + 3)
+      val ctx = new Ctx.Log with Ctx.Home {
+        val log = PrintLogger(
+          colored == "true",
+          if(colored == "true") Colors.Default
+          else Colors.BlackWhite,
+          System.out,
+          System.err,
+          System.err,
+          System.in
+        )
+        val home = Path(homeStr)
+      }
+      val result = runTests(
+        frameworkInstances = TestRunner.frameworks(frameworks),
+        entireClasspath = Agg.from(classpath.map(Path(_))),
+        testClassfilePath = Agg(Path(testCp)),
+        args = arguments
+      )(ctx)
+
+      // Clear interrupted state in case some badly-behaved test suite
+      // dirtied the thread-interrupted flag and forgot to clean up. Otherwise
+      // that flag causes writing the results to disk to fail
+      Thread.interrupted()
+      ammonite.ops.write(Path(outputPath), upickle.default.write(result))
+    }catch{case e: Throwable =>
+      println(e)
+      e.printStackTrace()
+    }
+    // Tests are over, kill the JVM whether or not anyone's threads are still running
+    // Always return 0, even if tests fail. The caller can pick up the detailed test
+    // results from the outputPath
+    System.exit(0)
+  }
+
+  def runTests(frameworkInstances: ClassLoader => Seq[sbt.testing.Framework],
+               entireClasspath: Agg[Path],
+               testClassfilePath: Agg[Path],
+               args: Seq[String])
+              (implicit ctx: Ctx.Log with Ctx.Home): (String, Seq[mill.scalalib.TestRunner.Result]) = {
+    Jvm.inprocess(entireClasspath, classLoaderOverrideSbtTesting = true, isolated = true, cl => {
+      val frameworks = frameworkInstances(cl)
+
+      val events = mutable.Buffer.empty[Event]
+
+      val doneMessages = frameworks.map{ framework =>
+        val runner = framework.runner(args.toArray, args.toArray, cl)
+
+        val testClasses = discoverTests(cl, framework, testClassfilePath)
+
+        val tasks = runner.tasks(
+          for ((cls, fingerprint) <- testClasses.toArray)
+            yield new TaskDef(cls.getName.stripSuffix("$"), fingerprint, true, Array(new SuiteSelector))
+        )
+
+        val taskQueue = tasks.to[mutable.Queue]
+        while (taskQueue.nonEmpty){
+          val next = taskQueue.dequeue().execute(
+            new EventHandler {
+              def handle(event: Event) = events.append(event)
+            },
+            Array(
+              new Logger {
+                def debug(msg: String) = ctx.log.outputStream.println(msg)
+
+                def error(msg: String) = ctx.log.outputStream.println(msg)
+
+                def ansiCodesSupported() = true
+
+                def warn(msg: String) = ctx.log.outputStream.println(msg)
+
+                def trace(t: Throwable) = t.printStackTrace(ctx.log.outputStream)
+
+                def info(msg: String) = ctx.log.outputStream.println(msg)
+              })
+          )
+          taskQueue.enqueue(next:_*)
+        }
+        runner.done()
+      }
+
+      val results = for(e <- events) yield {
+        val ex = if (e.throwable().isDefined) Some(e.throwable().get) else None
+        mill.scalalib.TestRunner.Result(
+          e.fullyQualifiedName(),
+          e.selector() match{
+            case s: NestedSuiteSelector => s.suiteId()
+            case s: NestedTestSelector => s.suiteId() + "." + s.testName()
+            case s: SuiteSelector => s.toString
+            case s: TestSelector => s.testName()
+            case s: TestWildcardSelector => s.testWildcard()
+          },
+          e.duration(),
+          e.status().toString,
+          ex.map(_.getClass.getName),
+          ex.map(_.getMessage),
+          ex.map(_.getStackTrace)
+        )
+      }
+
+      (doneMessages.mkString("\n"), results)
+    })
+  }
+
   def frameworks(frameworkNames: Seq[String])(cl: ClassLoader): Seq[sbt.testing.Framework] = {
     frameworkNames.map { name =>
       cl.loadClass(name).newInstance().asInstanceOf[sbt.testing.Framework]
@@ -19,5 +148,4 @@ object TestRunner {
   object Result{
     implicit def resultRW: upickle.default.ReadWriter[Result] = upickle.default.macroRW[Result]
   }
-
 }
diff --git a/scalalib/worker/src/mill/scalalib/worker/ScalaWorker.scala b/scalalib/worker/src/mill/scalalib/worker/ScalaWorker.scala
index cebef9b7..9b14757c 100644
--- a/scalalib/worker/src/mill/scalalib/worker/ScalaWorker.scala
+++ b/scalalib/worker/src/mill/scalalib/worker/ScalaWorker.scala
@@ -23,59 +23,6 @@ case class MockedLookup(am: File => Optional[CompileAnalysis]) extends PerClassp
     Locate.definesClass(classpathEntry)
 }
 
-object ScalaWorker{
-
-  def main(args: Array[String]): Unit = {
-    try{
-      var i = 0
-      def readArray() = {
-        val count = args(i).toInt
-        val slice = args.slice(i + 1, i + count + 1)
-        i = i + count + 1
-        slice
-      }
-      val frameworks = readArray()
-      val classpath = readArray()
-      val arguments = readArray()
-      val outputPath = args(i + 0)
-      val colored = args(i + 1)
-      val testCp = args(i + 2)
-      val homeStr = args(i + 3)
-      val ctx = new Ctx.Log with Ctx.Home {
-        val log = PrintLogger(
-          colored == "true",
-          if(colored == "true") Colors.Default
-          else Colors.BlackWhite,
-          System.out,
-          System.err,
-          System.err,
-          System.in
-        )
-        val home = Path(homeStr)
-      }
-      val result = Lib.runTests(
-        frameworkInstances = TestRunner.frameworks(frameworks),
-        entireClasspath = Agg.from(classpath.map(Path(_))),
-        testClassfilePath = Agg(Path(testCp)),
-        args = arguments
-      )(ctx)
-
-      // Clear interrupted state in case some badly-behaved test suite
-      // dirtied the thread-interrupted flag and forgot to clean up. Otherwise
-      // that flag causes writing the results to disk to fail
-      Thread.interrupted()
-      ammonite.ops.write(Path(outputPath), upickle.default.write(result))
-    }catch{case e: Throwable =>
-      println(e)
-      e.printStackTrace()
-    }
-    // Tests are over, kill the JVM whether or not anyone's threads are still running
-    // Always return 0, even if tests fail. The caller can pick up the detailed test
-    // results from the outputPath
-    System.exit(0)
-  }
-}
-
 class ScalaWorker(ctx0: mill.util.Ctx,
                   compilerBridgeClasspath: Array[String]) extends mill.scalalib.ScalaWorkerApi{
   @volatile var scalaClassloaderCache = Option.empty[(Long, ClassLoader)]
diff --git a/scratch/build.sc b/scratch/build.sc
index 9842039e..db446c99 100644
--- a/scratch/build.sc
+++ b/scratch/build.sc
@@ -1,4 +1,15 @@
-def thingy = T {
-  val blob = (('0' to '9') ++ ('a' to 'z') ++ ('A' to 'Z')).mkString
-  while(true) print(blob)
+import mill.Agg
+import mill.scalalib._
+
+trait JUnitTests extends TestModule{
+  def testFrameworks = Seq("com.novocode.junit.JUnitFramework")
+  def ivyDeps = Agg(ivy"com.novocode:junit-interface:0.11")
+}
+
+object core extends JavaModule{
+  object test extends Tests with JUnitTests
+}
+object app extends JavaModule{
+  def moduleDeps = Seq(core)
+  object test extends Tests with JUnitTests
 }
\ No newline at end of file