Add a `visualize` command to render portions of the build graph as SVG (#349)

* Transitive reduction of visualized graph via jgrapht now works

```
out/dev/launcher/dest/run -i visualize __.compile _
out/dev/launcher/dest/run -i visualize core.__
```

* Move test running logic from scalaworker into scalalib

This is to try and reduce the size of the classpath we are passing to the test runner subprocess, in an attempt to fix the command-line-too-long errors we're getting in Appveyor. Now the test runner subprocess should no longer need Zinc or all of it's transitive dependencies

* - Break out `GraphvizTools` into a separate Mill module, to avoid bloating the main jar and try to shorten the `MILL_SCALA_WORKER` classpath being sent to scalajslib.test (which is blowing up on windows as the CLI command is too long)

- Move the meat of `resolveDependencies` from `scalalib` to `main`, to support resolving mill modules

- DRY up resolution of mill modules in `Util.millProjectModule`

* fix mill module resolution

* Tweaks to try and make zinc work again...

* Tweak `Module#reflect` to try and make it happy with `visualize` module...

* fix integration test classpath

* move visualization into it's own module

4 files changed, 151 insertions, 147 deletions

diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala
index a05e1778..26c58a8b 100644
--- a/scalalib/src/mill/scalalib/JavaModule.scala
+++ b/scalalib/src/mill/scalalib/JavaModule.scala
@@ -303,8 +303,8 @@ trait TestModule extends JavaModule with TaskModule {
     val outputPath = T.ctx().dest/"out.json"
 
     Jvm.subprocess(
-      mainClass = "mill.scalalib.worker.ScalaWorker",
-      classPath = ScalaWorkerModule.classpath(),
+      mainClass = "mill.scalalib.TestRunner",
+      classPath = ScalaWorkerModule.scalalibClasspath().map(_.path),
       jvmArgs = forkArgs(),
       envArgs = forkEnv(),
       mainArgs =
@@ -330,7 +330,7 @@ trait TestModule extends JavaModule with TaskModule {
   def testLocal(args: String*) = T.command{
     val outputPath = T.ctx().dest/"out.json"
 
-    Lib.runTests(
+    TestRunner.runTests(
       TestRunner.frameworks(testFrameworks()),
       runClasspath().map(_.path),
       Agg(compile().classes.path),
diff --git a/scalalib/src/mill/scalalib/Lib.scala b/scalalib/src/mill/scalalib/Lib.scala
index bda8e8f8..7bd0a5ff 100644
--- a/scalalib/src/mill/scalalib/Lib.scala
+++ b/scalalib/src/mill/scalalib/Lib.scala
@@ -109,23 +109,12 @@ object Lib{
                                   deps: TraversableOnce[Dep],
                                   mapDependencies: Option[Dependency => Dependency] = None) = {
     val depSeq = deps.toSeq
-    val flattened = depSeq.map(depToDependency)
-
-    val forceVersions = depSeq.filter(_.force)
-      .map(depToDependency)
-      .map(mapDependencies.getOrElse(identity[Dependency](_)))
-      .map{d => d.module -> d.version}
-      .toMap
-
-    val start = Resolution(
-      flattened.map(mapDependencies.getOrElse(identity[Dependency](_))).toSet,
-      forceVersions = forceVersions,
-      mapDependencies = mapDependencies
+    mill.modules.Jvm.resolveDependenciesMetadata(
+      repositories,
+      depSeq.map(depToDependency),
+      depSeq.filter(_.force).map(depToDependency),
+      mapDependencies
     )
-
-    val fetch = Fetch.from(repositories, Cache.fetch())
-    val resolution = start.process.run(fetch).unsafePerformSync
-    (flattened, resolution)
   }
   /**
     * Resolve dependencies using Coursier.
@@ -139,54 +128,14 @@ object Lib{
                           deps: TraversableOnce[Dep],
                           sources: Boolean = false,
                           mapDependencies: Option[Dependency => Dependency] = None): Result[Agg[PathRef]] = {
-
-    val (_, resolution) = resolveDependenciesMetadata(
-      repositories, depToDependency, deps, mapDependencies
+    val depSeq = deps.toSeq
+    mill.modules.Jvm.resolveDependencies(
+      repositories,
+      depSeq.map(depToDependency),
+      depSeq.filter(_.force).map(depToDependency),
+      sources,
+      mapDependencies
     )
-    val errs = resolution.metadataErrors
-    if(errs.nonEmpty) {
-      val header =
-        s"""|
-            |Resolution failed for ${errs.length} modules:
-            |--------------------------------------------
-            |""".stripMargin
-
-      val errLines = errs.map {
-        case ((module, vsn), errMsgs) => s"  ${module.trim}:$vsn \n\t" + errMsgs.mkString("\n\t")
-      }.mkString("\n")
-      val msg = header + errLines + "\n"
-      Result.Failure(msg)
-    } else {
-
-      def load(artifacts: Seq[coursier.Artifact]) = {
-        val logger = None
-        val loadedArtifacts = scalaz.concurrent.Task.gatherUnordered(
-          for (a <- artifacts)
-            yield coursier.Cache.file(a, logger = logger).run
-              .map(a.isOptional -> _)
-        ).unsafePerformSync
-
-        val errors = loadedArtifacts.collect {
-          case (false, scalaz.-\/(x)) => x
-          case (true, scalaz.-\/(x)) if !x.notFound => x
-        }
-        val successes = loadedArtifacts.collect { case (_, scalaz.\/-(x)) => x }
-        (errors, successes)
-      }
-
-      val sourceOrJar =
-        if (sources) resolution.classifiersArtifacts(Seq("sources"))
-        else resolution.artifacts(true)
-      val (errors, successes) = load(sourceOrJar)
-      if(errors.isEmpty){
-        Agg.from(
-          successes.map(p => PathRef(Path(p), quick = true)).filter(_.path.ext == "jar")
-        )
-      }else{
-        val errorDetails = errors.map(e => s"${Util.newLine}  ${e.describe}").mkString
-        Result.Failure("Failed to load source dependencies" + errorDetails)
-      }
-    }
   }
   def scalaCompilerIvyDeps(scalaVersion: String) = Agg[Dep](
     ivy"org.scala-lang:scala-compiler:$scalaVersion".forceVersion(),
@@ -202,75 +151,6 @@ object Lib{
       force = false
     )
 
-  def runTests(frameworkInstances: ClassLoader => Seq[sbt.testing.Framework],
-               entireClasspath: Agg[Path],
-               testClassfilePath: Agg[Path],
-               args: Seq[String])
-              (implicit ctx: Ctx.Log with Ctx.Home): (String, Seq[mill.scalalib.TestRunner.Result]) = {
-    Jvm.inprocess(entireClasspath, classLoaderOverrideSbtTesting = true, cl => {
-      val frameworks = frameworkInstances(cl)
-
-      val events = mutable.Buffer.empty[Event]
-
-      val doneMessages = frameworks.map { framework =>
-        val runner = framework.runner(args.toArray, args.toArray, cl)
-
-        val testClasses = discoverTests(cl, framework, testClassfilePath)
-
-        val tasks = runner.tasks(
-          for ((cls, fingerprint) <- testClasses.toArray)
-          yield new TaskDef(cls.getName.stripSuffix("$"), fingerprint, true, Array(new SuiteSelector))
-        )
-
-        val taskQueue = tasks.to[mutable.Queue]
-        while (taskQueue.nonEmpty){
-          val next = taskQueue.dequeue().execute(
-            new EventHandler {
-              def handle(event: Event) = events.append(event)
-            },
-            Array(
-              new Logger {
-                def debug(msg: String) = ctx.log.outputStream.println(msg)
-
-                def error(msg: String) = ctx.log.outputStream.println(msg)
-
-                def ansiCodesSupported() = true
-
-                def warn(msg: String) = ctx.log.outputStream.println(msg)
-
-                def trace(t: Throwable) = t.printStackTrace(ctx.log.outputStream)
-
-                def info(msg: String) = ctx.log.outputStream.println(msg)
-              })
-          )
-          taskQueue.enqueue(next:_*)
-        }
-        ctx.log.outputStream.println(runner.done())
-      }
-
-      val results = for(e <- events) yield {
-        val ex = if (e.throwable().isDefined) Some(e.throwable().get) else None
-        mill.scalalib.TestRunner.Result(
-          e.fullyQualifiedName(),
-          e.selector() match{
-            case s: NestedSuiteSelector => s.suiteId()
-            case s: NestedTestSelector => s.suiteId() + "." + s.testName()
-            case s: SuiteSelector => s.toString
-            case s: TestSelector => s.testName()
-            case s: TestWildcardSelector => s.testWildcard()
-          },
-          e.duration(),
-          e.status().toString,
-          ex.map(_.getClass.getName),
-          ex.map(_.getMessage),
-          ex.map(_.getStackTrace)
-        )
-      }
-
-      (doneMessages.mkString("\n"), results)
-    })
-  }
-
   def listClassFiles(base: Path): Iterator[String] = {
     if (base.isDir) ls.rec(base).toIterator.filter(_.ext == "class").map(_.relativeTo(base).toString)
     else {
diff --git a/scalalib/src/mill/scalalib/ScalaWorkerApi.scala b/scalalib/src/mill/scalalib/ScalaWorkerApi.scala
index a01c1c2c..6865c541 100644
--- a/scalalib/src/mill/scalalib/ScalaWorkerApi.scala
+++ b/scalalib/src/mill/scalalib/ScalaWorkerApi.scala
@@ -22,20 +22,16 @@ trait ScalaWorkerModule extends mill.Module{
   )
 
   def classpath = T{
-    val scalaWorkerJar = sys.props("MILL_SCALA_WORKER")
-    if (scalaWorkerJar != null) {
-      mill.eval.Result.Success(Loose.Agg.from(scalaWorkerJar.split(',').map(Path(_))))
-    } else {
-      resolveDependencies(
-        repositories,
-        Lib.depToDependency(_, "2.12.4", ""),
-        Seq(ivy"com.lihaoyi::mill-scalalib-worker:${sys.props("MILL_VERSION")}")
-      ).map(_.map(_.path))
-    }
+    mill.modules.Util.millProjectModule("MILL_SCALA_WORKER", "mill-scalalib-worker", repositories)
   }
+
+  def scalalibClasspath = T{
+    mill.modules.Util.millProjectModule("MILL_SCALA_LIB", "mill-scalalib", repositories)
+  }
+
   def worker: Worker[ScalaWorkerApi] = T.worker{
     val cl = mill.util.ClassLoader.create(
-      classpath().map(_.toNIO.toUri.toURL).toVector,
+      classpath().map(_.path.toNIO.toUri.toURL).toVector,
       getClass.getClassLoader
     )
     val cls = cl.loadClass("mill.scalalib.worker.ScalaWorker")
diff --git a/scalalib/src/mill/scalalib/TestRunner.scala b/scalalib/src/mill/scalalib/TestRunner.scala
index 7bee681f..a014df0c 100644
--- a/scalalib/src/mill/scalalib/TestRunner.scala
+++ b/scalalib/src/mill/scalalib/TestRunner.scala
@@ -1,7 +1,136 @@
 package mill.scalalib
+import ammonite.ops.Path
+import ammonite.util.Colors
+import mill.Agg
+import mill.modules.Jvm
+import mill.scalalib.Lib.discoverTests
+import mill.util.{Ctx, PrintLogger}
 import mill.util.JsonFormatters._
+import sbt.testing._
+
+import scala.collection.mutable
 object TestRunner {
 
+
+  def main(args: Array[String]): Unit = {
+    try{
+      var i = 0
+      def readArray() = {
+        val count = args(i).toInt
+        val slice = args.slice(i + 1, i + count + 1)
+        i = i + count + 1
+        slice
+      }
+      val frameworks = readArray()
+      val classpath = readArray()
+      val arguments = readArray()
+      val outputPath = args(i + 0)
+      val colored = args(i + 1)
+      val testCp = args(i + 2)
+      val homeStr = args(i + 3)
+      val ctx = new Ctx.Log with Ctx.Home {
+        val log = PrintLogger(
+          colored == "true",
+          if(colored == "true") Colors.Default
+          else Colors.BlackWhite,
+          System.out,
+          System.err,
+          System.err,
+          System.in
+        )
+        val home = Path(homeStr)
+      }
+      val result = runTests(
+        frameworkInstances = TestRunner.frameworks(frameworks),
+        entireClasspath = Agg.from(classpath.map(Path(_))),
+        testClassfilePath = Agg(Path(testCp)),
+        args = arguments
+      )(ctx)
+
+      // Clear interrupted state in case some badly-behaved test suite
+      // dirtied the thread-interrupted flag and forgot to clean up. Otherwise
+      // that flag causes writing the results to disk to fail
+      Thread.interrupted()
+      ammonite.ops.write(Path(outputPath), upickle.default.write(result))
+    }catch{case e: Throwable =>
+      println(e)
+      e.printStackTrace()
+    }
+    // Tests are over, kill the JVM whether or not anyone's threads are still running
+    // Always return 0, even if tests fail. The caller can pick up the detailed test
+    // results from the outputPath
+    System.exit(0)
+  }
+
+  def runTests(frameworkInstances: ClassLoader => Seq[sbt.testing.Framework],
+               entireClasspath: Agg[Path],
+               testClassfilePath: Agg[Path],
+               args: Seq[String])
+              (implicit ctx: Ctx.Log with Ctx.Home): (String, Seq[mill.scalalib.TestRunner.Result]) = {
+    Jvm.inprocess(entireClasspath, classLoaderOverrideSbtTesting = true, isolated = true, cl => {
+      val frameworks = frameworkInstances(cl)
+
+      val events = mutable.Buffer.empty[Event]
+
+      val doneMessages = frameworks.map{ framework =>
+        val runner = framework.runner(args.toArray, args.toArray, cl)
+
+        val testClasses = discoverTests(cl, framework, testClassfilePath)
+
+        val tasks = runner.tasks(
+          for ((cls, fingerprint) <- testClasses.toArray)
+            yield new TaskDef(cls.getName.stripSuffix("$"), fingerprint, true, Array(new SuiteSelector))
+        )
+
+        val taskQueue = tasks.to[mutable.Queue]
+        while (taskQueue.nonEmpty){
+          val next = taskQueue.dequeue().execute(
+            new EventHandler {
+              def handle(event: Event) = events.append(event)
+            },
+            Array(
+              new Logger {
+                def debug(msg: String) = ctx.log.outputStream.println(msg)
+
+                def error(msg: String) = ctx.log.outputStream.println(msg)
+
+                def ansiCodesSupported() = true
+
+                def warn(msg: String) = ctx.log.outputStream.println(msg)
+
+                def trace(t: Throwable) = t.printStackTrace(ctx.log.outputStream)
+
+                def info(msg: String) = ctx.log.outputStream.println(msg)
+              })
+          )
+          taskQueue.enqueue(next:_*)
+        }
+        runner.done()
+      }
+
+      val results = for(e <- events) yield {
+        val ex = if (e.throwable().isDefined) Some(e.throwable().get) else None
+        mill.scalalib.TestRunner.Result(
+          e.fullyQualifiedName(),
+          e.selector() match{
+            case s: NestedSuiteSelector => s.suiteId()
+            case s: NestedTestSelector => s.suiteId() + "." + s.testName()
+            case s: SuiteSelector => s.toString
+            case s: TestSelector => s.testName()
+            case s: TestWildcardSelector => s.testWildcard()
+          },
+          e.duration(),
+          e.status().toString,
+          ex.map(_.getClass.getName),
+          ex.map(_.getMessage),
+          ex.map(_.getStackTrace)
+        )
+      }
+
+      (doneMessages.mkString("\n"), results)
+    })
+  }
+
   def frameworks(frameworkNames: Seq[String])(cl: ClassLoader): Seq[sbt.testing.Framework] = {
     frameworkNames.map { name =>
       cl.loadClass(name).newInstance().asInstanceOf[sbt.testing.Framework]
@@ -19,5 +148,4 @@ object TestRunner {
   object Result{
     implicit def resultRW: upickle.default.ReadWriter[Result] = upickle.default.macroRW[Result]
   }
-
 }