Merge branch 'master' of github.com:lihaoyi/mill

23 files changed, 734 insertions, 107 deletions

diff --git a/build.sc b/build.sc
index 825c6c10..969a3287 100755
--- a/build.sc
+++ b/build.sc
@@ -250,6 +250,10 @@ object contrib extends MillModule {
     }
    }
 
+  object tut extends MillModule {
+    def moduleDeps = Seq(scalalib)
+    def testArgs = Seq("-DMILL_VERSION=" + build.publishVersion()._2)
+  }
 }
 
 
@@ -362,7 +366,7 @@ def launcherScript(shellJvmArgs: Seq[String],
 }
 
 object dev extends MillModule{
-  def moduleDeps = Seq(scalalib, scalajslib, scalanativelib, contrib.scalapblib)
+  def moduleDeps = Seq(scalalib, scalajslib, scalanativelib, contrib.scalapblib, contrib.tut)
 
   def forkArgs =
     (
diff --git a/contrib/tut/src/mill/contrib/tut/TutModule.scala b/contrib/tut/src/mill/contrib/tut/TutModule.scala
new file mode 100644
index 00000000..f051a465
--- /dev/null
+++ b/contrib/tut/src/mill/contrib/tut/TutModule.scala
@@ -0,0 +1,132 @@
+package mill
+package contrib.tut
+
+import ammonite.ops._
+import coursier.MavenRepository
+import mill.scalalib._
+import scala.util.matching.Regex
+
+/**
+  * Tut is a documentation tool which compiles and evaluates Scala code in documentation files and provides various options for configuring how the results will be displayed in the compiled documentation.
+  *
+  * Extending this trait declares a Scala module which compiles markdown, HTML and `.txt` files in the `tut` folder of the module with Tut.
+  *
+  * By default the resulting documents are simply placed in the Mill build output folder but they can be placed elsewhere by overriding the [[mill.contrib.tut.TutModule#tutTargetDirectory]] task.
+  *
+  * For example:
+  *
+  * {{{
+  * // build.sc
+  * import mill._, scalalib._, contrib.tut.__
+  *
+  * object example extends TutModule {
+  *   def scalaVersion = "2.12.6"
+  *   def tutVersion = "0.6.7"
+  * }
+  * }}}
+  *
+  * This defines a project with the following layout:
+  *
+  * {{{
+  * build.sc
+  * example/
+  *     src/
+  *     tut/
+  *     resources/
+  * }}}
+  *
+  * In order to compile documentation we can execute the `tut` task in the module:
+  *
+  * {{{
+  * sh> mill example.tut
+  * }}}
+  */
+trait TutModule extends ScalaModule {
+  /**
+    * This task determines where documentation files must be placed in order to be compiled with Tut. By default this is the `tut` folder at the root of the module.
+    */
+  def tutSourceDirectory = T.sources { millSourcePath / 'tut }
+
+  /**
+    * A task which determines where the compiled documentation files will be placed. By default this is simply the Mill build's output folder for this task,
+    * but this can be reconfigured so that documentation goes to the root of the module (e.g. `millSourcePath`) or to a dedicated folder (e.g. `millSourcePath / 'docs`)
+    */
+  def tutTargetDirectory: T[Path] = T { T.ctx().dest }
+
+  /**
+    * A task which determines what classpath is used when compiling documentation. By default this is configured to use the same inputs as the [[mill.contrib.tut.TutModule#runClasspath]],
+    * except for using [[mill.contrib.tut.TutModule#tutIvyDeps]] rather than the module's [[mill.contrib.tut.TutModule#runIvyDeps]].
+    */
+  def tutClasspath: T[Agg[PathRef]] = T {
+    // Same as runClasspath but with tut added to ivyDeps from the start
+    // This prevents duplicate, differently versioned copies of scala-library ending up on the classpath which can happen when resolving separately
+    transitiveLocalClasspath() ++
+    resources() ++
+    localClasspath() ++
+    unmanagedClasspath() ++
+    tutIvyDeps()
+  }
+
+  /**
+    * A task which determines the scalac plugins which will be used when compiling code examples with Tut. The default is to use the [[mill.contrib.tut.TutModule#scalacPluginIvyDeps]] for the module.
+    */
+  def tutScalacPluginIvyDeps: T[Agg[Dep]] = scalacPluginIvyDeps()
+
+  /**
+    * A [[scala.util.matching.Regex]] task which will be used to determine which files should be compiled with tut. The default pattern is as follows: `.*\.(md|markdown|txt|htm|html)`.
+    */
+  def tutNameFilter: T[Regex] = T { """.*\.(md|markdown|txt|htm|html)""".r }
+
+  /**
+    * The scalac options which will be used when compiling code examples with Tut. The default is to use the [[mill.contrib.tut.TutModule#scalacOptions]] for the module,
+    * but filtering out options which are problematic in the REPL, e.g. `-Xfatal-warnings`, `-Ywarn-unused-imports`.
+    */
+  def tutScalacOptions: T[Seq[String]] =
+    scalacOptions().filterNot(Set(
+      "-Ywarn-unused:imports",
+      "-Ywarn-unused-import",
+      "-Ywarn-dead-code",
+      "-Xfatal-warnings"
+    ))
+
+  /**
+    * The version of Tut to use.
+    */
+  def tutVersion: T[String]
+
+  /**
+    * A task which determines how to fetch the Tut jar file and all of the dependencies required to compile documentation for the module and returns the resulting files.
+    */
+  def tutIvyDeps: T[Agg[PathRef]] = T {
+    Lib.resolveDependencies(
+      repositories :+ MavenRepository(s"https://dl.bintray.com/tpolecat/maven"),
+      Lib.depToDependency(_, scalaVersion()),
+      compileIvyDeps() ++ transitiveIvyDeps() ++ Seq(
+        ivy"org.tpolecat::tut-core:${tutVersion()}"
+      )
+    )
+  }
+
+  /**
+    * A task which performs the dependency resolution for the scalac plugins to be used with Tut.
+    */
+  def tutPluginJars: T[Agg[PathRef]] = resolveDeps(tutScalacPluginIvyDeps)()
+
+  /**
+    * Run Tut using the configuration specified in this module. The working directory used is the [[mill.contrib.tut.TutModule#millSourcePath]].
+    */
+  def tut: T[CommandResult] = T {
+    val in = tutSourceDirectory().head.path.toIO.getAbsolutePath
+    val out = tutTargetDirectory().toIO.getAbsolutePath
+    val re = tutNameFilter()
+    val opts = tutScalacOptions()
+    val pOpts = tutPluginJars().map(pathRef => "-Xplugin:" + pathRef.path.toIO.getAbsolutePath)
+    val tutArgs = List(in, out, re.pattern.toString) ++ opts ++ pOpts
+    %%(
+      'java,
+      "-cp", tutClasspath().map(_.path.toIO.getAbsolutePath).mkString(java.io.File.pathSeparator),
+      "tut.TutMain",
+      tutArgs
+    )(wd = millSourcePath)
+  }
+}
diff --git a/contrib/tut/test/src/mill/contrib/tut/TutTests.scala b/contrib/tut/test/src/mill/contrib/tut/TutTests.scala
new file mode 100644
index 00000000..fd369eed
--- /dev/null
+++ b/contrib/tut/test/src/mill/contrib/tut/TutTests.scala
@@ -0,0 +1,124 @@
+package mill.contrib
+package tut
+
+import ammonite.ops._
+import mill._
+import mill.eval.Result._
+import mill.scalalib._
+import mill.util.{TestEvaluator, TestUtil}
+import utest._
+import utest.framework.TestPath
+
+object TutTests extends TestSuite {
+
+  trait TutTestModule extends TestUtil.BaseModule with TutModule {
+    def millSourcePath = TestUtil.getSrcPathBase() / millOuterCtx.enclosing.split('.')
+    def scalaVersion = "2.12.4"
+    def tutVersion = "0.6.7"
+  }
+
+  object TutTest extends TutTestModule
+
+  object TutCustomTest extends TutTestModule {
+    def tutTargetDirectory = millSourcePath
+  }
+
+  object TutLibrariesTest extends TutTestModule {
+    def ivyDeps = Agg(ivy"org.typelevel::cats-core:1.4.0")
+    def tutSourceDirectory = T.sources { resourcePathWithLibraries }
+    def scalacPluginIvyDeps = Agg(ivy"org.spire-math::kind-projector:0.9.8")
+  }
+
+  val resourcePath = pwd / 'contrib / 'tut / 'test / 'tut
+  val resourcePathWithLibraries = pwd / 'contrib / 'tut / 'test / "tut-with-libraries"
+
+  def workspaceTest[T](m: TestUtil.BaseModule, resourcePath: Path = resourcePath)
+                      (t: TestEvaluator => T)
+                      (implicit tp: TestPath): T = {
+    val eval = new TestEvaluator(m)
+    rm(m.millSourcePath)
+    rm(eval.outPath)
+    mkdir(m.millSourcePath)
+    cp(resourcePath, m.millSourcePath / 'tut)
+    t(eval)
+  }
+
+  def tests: Tests = Tests {
+    'tut - {
+      'createOutputFile - workspaceTest(TutTest) { eval =>
+        val expectedPath =
+          eval.outPath / 'tutTargetDirectory / 'dest / "TutExample.md"
+
+        val expected =
+          """
+          |```scala
+          |scala> 1 + 1
+          |res0: Int = 2
+          |```
+          |
+          """.trim.stripMargin
+
+        val Right((result, evalCount)) = eval.apply(TutTest.tut)
+
+        assert(
+          exists(expectedPath) &&
+            read! expectedPath == expected
+        )
+      }
+
+      'supportCustomSettings - workspaceTest(TutCustomTest) { eval =>
+        val defaultPath =
+          eval.outPath / 'tutTargetDirectory / 'dest / "TutExample.md"
+        val expectedPath =
+          TutCustomTest.millSourcePath / "TutExample.md"
+
+        val expected =
+          """
+          |```scala
+          |scala> 1 + 1
+          |res0: Int = 2
+          |```
+          |
+          """.trim.stripMargin
+
+        val Right((result, evalCount)) = eval.apply(TutCustomTest.tut)
+
+        assert(
+          !exists(defaultPath) &&
+            exists(expectedPath) &&
+            read! expectedPath == expected
+        )
+      }
+
+      'supportUsingLibraries - workspaceTest(TutLibrariesTest, resourcePath = resourcePathWithLibraries) { eval =>
+        val expectedPath =
+          eval.outPath / 'tutTargetDirectory / 'dest / "TutWithLibraries.md"
+
+        val expected =
+          """
+            |```scala
+            |import cats._
+            |import cats.arrow.FunctionK
+            |import cats.implicits._
+            |```
+            |
+            |```scala
+            |scala> List(1, 2, 3).combineAll
+            |res0: Int = 6
+            |
+            |scala> λ[FunctionK[List, Option]](_.headOption)(List(1, 2 ,3))
+            |res1: Option[Int] = Some(1)
+            |```
+            |
+          """.trim.stripMargin
+
+        val Right(_) = eval.apply(TutLibrariesTest.tut)
+
+        assert(
+          exists(expectedPath) &&
+            read! expectedPath == expected
+        )
+      }
+    }
+  }
+}
diff --git a/contrib/tut/test/tut-with-libraries/TutWithLibraries.md b/contrib/tut/test/tut-with-libraries/TutWithLibraries.md
new file mode 100644
index 00000000..7df1e703
--- /dev/null
+++ b/contrib/tut/test/tut-with-libraries/TutWithLibraries.md
@@ -0,0 +1,10 @@
+```tut:silent
+import cats._
+import cats.arrow.FunctionK
+import cats.implicits._
+```
+
+```tut
+List(1, 2, 3).combineAll
+λ[FunctionK[List, Option]](_.headOption)(List(1, 2 ,3))
+```
diff --git a/contrib/tut/test/tut/TutExample.md b/contrib/tut/test/tut/TutExample.md
new file mode 100644
index 00000000..c149fbc6
--- /dev/null
+++ b/contrib/tut/test/tut/TutExample.md
@@ -0,0 +1,3 @@
+```tut
+1 + 1
+```
diff --git a/docs/pages/4 - Tasks.md b/docs/pages/4 - Tasks.md
index 6c7737e0..e8a1c94c 100644
--- a/docs/pages/4 - Tasks.md
+++ b/docs/pages/4 - Tasks.md
@@ -199,6 +199,9 @@ task are streamed to standard out/error as you would expect, but each task's
 specific output is also streamed to a log file on disk, e.g. `out/run/log` or
 `out/classFiles/log` for you to inspect later.
 
+Messages logged with `log.debug` appear by default only in the log files.
+You can use the `--debug` option when running mill to show them on the console too.
+
 ### mill.util.Ctx.Env
 
 - `T.ctx().env`
diff --git a/docs/pages/7 - Extending Mill.md b/docs/pages/7 - Extending Mill.md
index 2eb7c93b..2e59dfe3 100644
--- a/docs/pages/7 - Extending Mill.md
+++ b/docs/pages/7 - Extending Mill.md
@@ -176,7 +176,6 @@ def idea(ev: Evaluator) = T.command {
 ```
 
 Many built-in tools are implemented as custom evaluator commands:
-[all](intro.html#all), [inspect](intro.html#inspect),
-[resolve](intro.html#resolve), [show](intro.html#show). If you want a way to run Mill
-commands and programmatically manipulate the tasks and outputs, you do so with
-your own evaluator command.
+[all](http://www.lihaoyi.com/mill/#all), [inspect](http://www.lihaoyi.com/mill/#inspect),
+[resolve](http://www.lihaoyi.com/mill/#resolve), [show](http://www.lihaoyi.com/mill/#show).
+If you want a way to run Mill commands and programmatically manipulate the tasks and outputs, you do so with your own evaluator command.
diff --git a/docs/pages/9 - Contrib Modules.md b/docs/pages/9 - Contrib Modules.md
index f6c9cc7b..cd02f2c2 100644
--- a/docs/pages/9 - Contrib Modules.md
+++ b/docs/pages/9 - Contrib Modules.md
@@ -1,5 +1,36 @@
 ## Contrib Modules
 
+### BuildInfo
+
+Generate scala code from your buildfile.
+This plugin generates a single object containing information from your build.
+
+To declare a module that uses BuildInfo you must extend the `mill.contrib.BuildInfo` trait when defining your module.
+
+Quickstart:
+```scala
+object project extends BuildInfo {
+  val name = "poject-name"
+  def  buildInfoMembers: T[Map[String, String]] = T {
+    Map(
+      "name" -> name),
+      "scalaVersion" -> scalaVersion()
+    )
+  }
+}
+```
+
+#### Configuration options
+
+* `def buildInfoMembers: T[Map[String, String]]`
+  The map containing all member names and values for the generated info object.
+
+* `def buildInfoObjectName: String`, default: `BuildInfo`
+  The name of the object which contains all the members from `buildInfoMembers`.
+
+* `def buildInfoPackageName: Option[String]`, default: `None`
+  The package name of the object.
+
 ### ScalaPB
 
 This module allows [ScalaPB](https://scalapb.github.io) to be used in Mill builds. ScalaPB is a [Protocol Buffers](https://developers.google.com/protocol-buffers/) compiler plugin that generates Scala case classes, encoders and decoders for protobuf messages.
@@ -50,96 +81,323 @@ object example extends ScalaPBModule {
 }
 ```
 
-### BuildInfo
+### TestNG
 
-Generate scala code from your buildfile.  
-This plugin generates a single object containing information from your build.
-  
-To declare a module that uses BuildInfo you must extend the `mill.contrib.BuildInfo` trait when defining your module.
+Provides support for [TestNG](https://testng.org/doc/index.html).
 
-Quickstart:
-  ```scala
-  object project extends BuildInfo {
-    val name = "poject-name"
-    def  buildInfoMembers: T[Map[String, String]] = T {
-      Map(
-        "name" -> name),
-        "scalaVersion" -> scalaVersion()
-      )
-    }
+To use TestNG as test framework, you need to add it to the `TestModule.testFrameworks` property.
+
+```scala
+object project extends ScalaModule {
+  object test extends Tests{
+    def testFrameworks = Seq("mill.testng.TestNGFramework")
   }
-  ```
-  
+}
+```
+
+### Tut
+
+This module allows [Tut](https://tpolecat.github.io/tut) to be used in Mill builds. Tut is a documentation tool which compiles and evaluates Scala code in documentation files and provides various options for configuring how the results will be displayed in the compiled documentation.
+
+To declare a module that uses Tut you can extend the `mill.contrib.tut.TutModule` trait when defining your module.
+
+This creates a Scala module which compiles markdown, HTML and `.txt` files in the `tut` folder of the module with Tut.
+
+By default the resulting documents are simply placed in the Mill build output folder but they can be placed elsewhere by overriding the `tutTargetDirectory` task.
+
+```scala
+// build.sc
+import mill._, scalalib._, contrib.tut.__
+
+object example extends TutModule {
+  def scalaVersion = "2.12.6"
+  def tutVersion = "0.6.7"
+}
+```
+
+This defines a project with the following layout:
+
+```
+build.sc
+example/
+    src/
+    tut/
+    resources/
+```
+
+In order to compile documentation we can execute the `tut` task in the module:
+
+```
+sh> mill example.tut
+```
+
 #### Configuration options
 
-* `def buildInfoMembers: T[Map[String, String]]`  
-  The map containing all member names and values for the generated info object.
+* tutSourceDirectory - This task determines where documentation files must be placed in order to be compiled with Tut. By default this is the `tut` folder at the root of the module.
 
-* `def buildInfoObjectName: String`, default: `BuildInfo`  
-  The name of the object which contains all the members from `buildInfoMembers`.
+* tutTargetDirectory - A task which determines where the compiled documentation files will be placed. By default this is simply the Mill build's output folder for the `tutTargetDirectory` task but this can be reconfigured so that documentation goes to the root of the module (e.g. `millSourcePath`) or to a dedicated folder (e.g. `millSourcePath / 'docs`)
 
-* `def buildInfoPackageName: Option[String]`, default: `None`  
-  The package name of the object.
+* tutClasspath - A task which determines what classpath is used when compiling documentation. By default this is configured to use the same inputs as the `runClasspath`, except for using `tutIvyDeps` rather than the module's `ivyDeps`.
+
+* tutScalacPluginIvyDeps - A task which determines the scalac plugins which will be used when compiling code examples with Tut. The default is to use the `scalacPluginIvyDeps` for the module.
+
+* tutNameFilter - A `scala.util.matching.Regex` task which will be used to determine which files should be compiled with tut. The default pattern is as follows: `.*\.(md|markdown|txt|htm|html)`.
+
+* tutScalacOptions - The scalac options which will be used when compiling code examples with Tut. The default is to use the `scalacOptions` for the module but filtering out options which are problematic in the REPL, e.g. `-Xfatal-warnings`, `-Ywarn-unused-imports`.
+
+* tutVersion - The version of Tut to use.
+
+* tutIvyDeps - A task which determines how to fetch the Tut jar file and all of the dependencies required to compile documentation for the module and returns the resulting files.
+
+* tutPluginJars - A task which performs the dependency resolution for the scalac plugins to be used with Tut.
+
+### Twirl
+
+Twirl templates support.
+
+To declare a module that needs to compile twirl templates you must extend the `mill.twirllib.TwirlModule` trait when defining your module. 
+Also note that twirl templates get compiled into scala code, so you also need to extend `ScalaModule`.
+ 
+```scala
+import $ivy.`com.lihaoyi::mill-contrib-twirllib:0.3.2`,  mill.twirllib._
+object app extends ScalaModule with TwirlModule {
+
+} 
+``` 
+
+#### Configuration options
+
+* ` def twirlVersion: T[String]` (mandatory) - the version of the twirl compiler to use, like "1.3.15"
+
+#### Details
+
+The following filesystem layout is expected:
+
+```text
+build.sc
+app/
+    views/
+        view1.scala.html
+        view2.scala.html
+```
+
+`TwirlModule` adds the `compileTwirl` task to the module:
+```
+mill app.compileTwirl
+```
+
+(it will be automatically run whenever you compile your module)
+
+This task will compile `*.scala.html` templates (and others, like `*.scala.txt`) into the `out/app/compileTwirl/dest` 
+directory. This directory must be added to the generated sources of the module to be compiled and made accessible from the rest of the code:
+```scala
+object app extends ScalaModule with TwirlModule {
+  def twirlVersion = "1.3.15"
+  def generatedSources = T{ Seq(compileTwirl().classes) }
+}
+``` 
 
+#### Caveats
 
-### Other Mill Plugins
+There is a couple of caveats, to be aware of, as of now (in `v0.3.2`).
 
-- [ensime](https://github.com/yyadavalli/mill-ensime "mill-ensime")
+##### Packages 
+First, if you structure your twirl templates into packages, like this:
+```text
+build.sc
+app/
+    src/hello/
+        Main.scala
+    views/
+        hello/
+            another/ 
+                view1.scala.html
+                view2.scala.html
+```
 
-  Create an [.ensime](http://ensime.github.io/ "ensime") file for your build.
-  
-  Quickstart:
-  ```scala
-  import $ivy.`fun.valycorp::mill-ensime:0.0.1`
-  ```
-  ```sh
-  sh> mill fun.valycorp.mill.GenEnsime/ensimeConfig
-  ```
-  
-- [dgraph](https://github.com/ajrnz/mill-dgraph "mill-dgraph")
+the generated sources in the `out` directory will look like this:
+```text
+build.sc
+out/app/compileTwirl/dest/
+    hello/
+      another/
+        html/
+          view1.template.scala
+          view2.template.scala
+```
 
-  Show transitive dependencies of your build in your browser.
-  
-  Quickstart:
-  ```scala
-  import $ivy.`com.github.ajrnz::mill-dgraph:0.2.0`
-  ```
-  ```sh
-  sh> mill plugin.dgraph.browseDeps(proj)()
-  ```
+Looking at the `mill show app.compileTwirl` in this setup shows this:
+```
+{
+    ...
+    "classes": "ref: ... : .../out/app/compileTwirl/dest/html"
+}
+```
 
-- [publishM2](https://github.com/lefou/mill-publishM2 "mill-publishM2")
+Basically it means that currently `TwirlModule` expects all templates to be html and with no packages.
+So adding this directly to the generated sources will not exactly work as expected (as there might not even be a `out/app/compileTwirl/dest/html` directory
+at all, unless you have templates in the default package).
 
-  Mill plugin to publish artifacts into a local Maven repository.
+The workaround is simple, though:
+```scala
+object app extends ScalaModule with TwirlModule {
+  def twirlVersion = "1.3.15"
+  override def generatedSources = T{
+    val classes = compileTwirl().classes
+    Seq(classes.copy(path = classes.path / up)) // we just move one dir up
+  }
+}
+``` 
+
+This should cover the problem with templates under packages, and also should make other-than-html 
+templates available as well.
+
+##### Default imports
 
-  Quickstart:
+Another problem is with some default imports that the twirl sbt plugin assumes, but it seems not to work with `TwirlModule`.
 
-  Just mix-in the `PublishM2Module` into your project.
-  `PublishM2Module` already extends mill's built-in `PublishModule`.
+If you reference `Html` in your templates, like
+
+```scala
+// wrapper.scala.html
+@(content: Html)
+<div class="wrapper">
+  @content
+</div>
+```
 
-  File: `build.sc`
-  ```scala
-  import mill._, scalalib._, publish._
+the template will not compile. You'll need to add this import:
+```
+@import play.twirl.api._
+```
 
-  import $ivy.`de.tototec::de.tobiasroeser.mill.publishM2:0.0.1`
-  import de.tobiasroeser.mill.publishM2._
+in the template that uses twirl classes.
 
-  object project extends PublishModule with PublishM2Module {
-    // ...
+Another one is `@defining`, which might be used like this:
+```
+@defining({
+  val calculatedClass = {
+    // do some calculations here
   }
-  ```
-
-  Publishing to default local Maven repository
-
-  ```
-  > mill project.publishM2Local
-  [40/40] project.publishM2Local
-  Publishing to /home/user/.m2/repository
-  ```
-
-  Publishing to custom local Maven repository
-  ```
-  > mill project.publishM2Local /tmp/m2repo
-  [40/40] project.publishM2Local
-  Publishing to /tmp/m2repo
-  ```
+  calculatedClass
+}) { calculatedClass =>
+    <div class="@calculatedClass">stuff 1</div>
+    <div class="@calculatedClass">stuff 2</div>
+}
+```
+
+You'll need this import:
+```scala
+@import play.twirl.api.TwirlFeatureImports._
+```
+
+At some point `TwirlModule` might get support for the additional "default" imports, which will make this much easier, 
+but right now it is unimplemented
+
+```scala
+  // REMIND currently it's not possible to override these default settings
+  private def twirlAdditionalImports: Seq[String] = Nil
+```
+
+#### Example
+There's an [example project](https://github.com/lihaoyi/cask/tree/master/example/twirl)
+
+
+
+## Thirdparty Mill Plugins
+
+### DGraph
+
+Show transitive dependencies of your build in your browser.
+
+Project home: https://github.com/ajrnz/mill-dgraph
+
+#### Quickstart
+
+```scala
+import $ivy.`com.github.ajrnz::mill-dgraph:0.2.0`
+```
+
+```sh
+sh> mill plugin.dgraph.browseDeps(proj)()
+```
+
+### Ensime
+
+Create an [.ensime](http://ensime.github.io/ "ensime") file for your build.
+
+Project home: https://github.com/yyadavalli/mill-ensime
+
+#### Quickstart
+
+```scala
+import $ivy.`fun.valycorp::mill-ensime:0.0.1`
+```
+
+```sh
+sh> mill fun.valycorp.mill.GenEnsime/ensimeConfig
+```
+
+### OSGi
+
+Produce OSGi Bundles with mill.
+
+Project home: https://github.com/lefou/mill-osgi
+
+#### Quickstart
+
+```scala
+import $ivy.`de.tototec::de.tobiasroeser.mill.osgi:0.0.2`
+import de.tobiasroeser.mill.osgi._
+
+object project extends ScalaModule with OsgiBundleModule {
+
+  def bundleSymbolicName = "com.example.project"
+
+  def osgiHeaders = T{ osgiHeaders().copy(
+    `Export-Package`   = Seq("com.example.api"),
+    `Bundle-Activator` = Some("com.example.internal.Activator")
+  )}
+
+}
+```
+
+
+### PublishM2
+
+Mill plugin to publish artifacts into a local Maven repository.
+
+Project home: https://github.com/lefou/mill-publishM2
+
+#### Quickstart
+
+Just mix-in the `PublishM2Module` into your project.
+`PublishM2Module` already extends mill's built-in `PublishModule`.
+
+File: `build.sc`
+```scala
+import mill._, scalalib._, publish._
+
+import $ivy.`de.tototec::de.tobiasroeser.mill.publishM2:0.0.1`
+import de.tobiasroeser.mill.publishM2._
+
+object project extends PublishModule with PublishM2Module {
+  // ...
+}
+```
+
+Publishing to default local Maven repository
+
+```bash
+> mill project.publishM2Local
+[40/40] project.publishM2Local
+Publishing to /home/user/.m2/repository
+```
+
+Publishing to custom local Maven repository
+
+```bash
+> mill project.publishM2Local /tmp/m2repo
+[40/40] project.publishM2Local
+Publishing to /tmp/m2repo
+```
diff --git a/main/core/src/mill/define/Graph.scala b/main/core/src/mill/define/Graph.scala
index f06dca11..3119f2fb 100644
--- a/main/core/src/mill/define/Graph.scala
+++ b/main/core/src/mill/define/Graph.scala
@@ -5,7 +5,14 @@ import mill.util.MultiBiMap
 import mill.util.Strict.Agg
 
 object Graph {
-  class TopoSorted private[Graph](val values: Agg[Task[_]])
+
+  /**
+    * The `values` [[Agg]] is guaranteed to be topological sorted and cycle free.
+    * That's why the constructor is package private.
+    * @see [[Graph.topoSorted]]
+    */
+  class TopoSorted private[Graph] (val values: Agg[Task[_]])
+
   def groupAroundImportantTargets[T](topoSortedTargets: TopoSorted)
                                     (important: PartialFunction[Task[_], T]): MultiBiMap[T, Task[_]] = {
 
@@ -27,6 +34,10 @@ object Graph {
     output
   }
 
+  /**
+   * Collects all transitive dependencies (targets) of the given targets,
+   * including the given targets.
+   */
   def transitiveTargets(sourceTargets: Agg[Task[_]]): Agg[Task[_]] = {
     val transitiveTargets = new Agg.Mutable[Task[_]]
     def rec(t: Task[_]): Unit = {
diff --git a/main/core/src/mill/eval/Evaluator.scala b/main/core/src/mill/eval/Evaluator.scala
index ded4afdd..2ffc469b 100644
--- a/main/core/src/mill/eval/Evaluator.scala
+++ b/main/core/src/mill/eval/Evaluator.scala
@@ -342,7 +342,7 @@ case class Evaluator(home: Path,
 
   def resolveLogger(logPath: Option[Path]): Logger = logPath match{
     case None => log
-    case Some(path) => MultiLogger(log.colored, log, FileLogger(log.colored, path))
+    case Some(path) => MultiLogger(log.colored, log, FileLogger(log.colored, path, debugEnabled = true))
   }
 }
 
diff --git a/main/core/src/mill/util/JsonFormatters.scala b/main/core/src/mill/util/JsonFormatters.scala
index f92941f7..2728d94d 100644
--- a/main/core/src/mill/util/JsonFormatters.scala
+++ b/main/core/src/mill/util/JsonFormatters.scala
@@ -3,6 +3,7 @@ package mill.util
 import ammonite.ops.{Bytes, Path}
 import upickle.Js
 import upickle.default.{ReadWriter => RW}
+import scala.util.matching.Regex
 object JsonFormatters extends JsonFormatters
 trait JsonFormatters {
   implicit val pathReadWrite: RW[ammonite.ops.Path] = upickle.default.readwriter[String]
@@ -11,6 +12,12 @@ trait JsonFormatters {
       Path(_)
     )
 
+  implicit val regexReadWrite: RW[Regex] = upickle.default.readwriter[String]
+    .bimap[Regex](
+      _.pattern.toString,
+      _.r
+    )
+
   implicit val bytesReadWrite: RW[Bytes] = upickle.default.readwriter[String]
     .bimap(
       o => javax.xml.bind.DatatypeConverter.printBase64Binary(o.array),
diff --git a/main/core/src/mill/util/Logger.scala b/main/core/src/mill/util/Logger.scala
index 37ae8577..1db66039 100644
--- a/main/core/src/mill/util/Logger.scala
+++ b/main/core/src/mill/util/Logger.scala
@@ -5,20 +5,23 @@ import java.io._
 import ammonite.ops.{Path, rm}
 import ammonite.util.Colors
 
-
 /**
   * The standard logging interface of the Mill build tool.
   *
-  * Contains four primary logging methods, in order of increasing importance:
+  * Contains these primary logging methods, in order of increasing importance:
+  *
+  * - `debug` : internal debug messages normally not shown to the user;
+  * mostly useful when debugging issues
   *
   * - `ticker`: short-lived logging output where consecutive lines over-write
-  *   each other; useful for information which is transient and disposable
+  * each other; useful for information which is transient and disposable
   *
   * - `info`: miscellaneous logging output which isn't part of the main output
-  *   a user is looking for, but useful to provide context on what Mill is doing
+  * a user is looking for, but useful to provide context on what Mill is doing
   *
   * - `error`: logging output which represents problems the user should care
-  *   about
+  * about
+  *
   *
   * Also contains the two forwarded stdout and stderr streams, for code executed
   * by Mill to use directly. Typically these correspond to the stdout and stderr,
@@ -27,23 +30,30 @@ import ammonite.util.Colors
   */
 trait Logger {
   def colored: Boolean
+
   val errorStream: PrintStream
   val outputStream: PrintStream
   val inStream: InputStream
+
   def info(s: String): Unit
   def error(s: String): Unit
   def ticker(s: String): Unit
+  def debug(s: String): Unit
+
   def close(): Unit = ()
 }
 
 object DummyLogger extends Logger {
   def colored = false
+
   object errorStream extends PrintStream(_ => ())
   object outputStream extends PrintStream(_ => ())
   val inStream = new ByteArrayInputStream(Array())
+
   def info(s: String) = ()
   def error(s: String) = ()
   def ticker(s: String) = ()
+  def debug(s: String) = ()
 }
 
 class CallbackStream(wrapped: OutputStream,
@@ -78,13 +88,17 @@ object PrintState{
   case object Newline extends PrintState
   case object Middle extends PrintState
 }
-case class PrintLogger(colored: Boolean,
-                       disableTicker: Boolean,
-                       colors: ammonite.util.Colors,
-                       outStream: PrintStream,
-                       infoStream: PrintStream,
-                       errStream: PrintStream,
-                       inStream: InputStream) extends Logger {
+
+case class PrintLogger(
+                        colored: Boolean,
+                        disableTicker: Boolean,
+                        colors: ammonite.util.Colors,
+                        outStream: PrintStream,
+                        infoStream: PrintStream,
+                        errStream: PrintStream,
+                        inStream: InputStream,
+                        debugEnabled: Boolean
+                      ) extends Logger {
 
   var printState: PrintState = PrintState.Newline
 
@@ -121,9 +135,14 @@ case class PrintLogger(colored: Boolean,
       printState = PrintState.Ticker
     }
   }
+
+  def debug(s: String) = if (debugEnabled) {
+    printState = PrintState.Newline
+    errStream.println(s)
+  }
 }
 
-case class FileLogger(colored: Boolean, file: Path) extends Logger {
+case class FileLogger(colored: Boolean, file: Path, debugEnabled: Boolean) extends Logger {
   private[this] var outputStreamUsed: Boolean = false
 
   lazy val outputStream = {
@@ -141,6 +160,7 @@ case class FileLogger(colored: Boolean, file: Path) extends Logger {
   def info(s: String) = outputStream.println(s)
   def error(s: String) = outputStream.println(s)
   def ticker(s: String) = outputStream.println(s)
+  def debug(s: String) = if (debugEnabled) outputStream.println(s)
   val inStream: InputStream = DummyInputStream
   override def close() = {
     if (outputStreamUsed)
@@ -198,6 +218,11 @@ case class MultiLogger(colored: Boolean, logger1: Logger, logger2: Logger) exten
     logger2.ticker(s)
   }
 
+  def debug(s: String) = {
+    logger1.debug(s)
+    logger2.debug(s)
+  }
+
   override def close() = {
     logger1.close()
     logger2.close()
diff --git a/main/core/src/mill/util/MultiBiMap.scala b/main/core/src/mill/util/MultiBiMap.scala
index 2cb81944..73bb42c4 100644
--- a/main/core/src/mill/util/MultiBiMap.scala
+++ b/main/core/src/mill/util/MultiBiMap.scala
@@ -2,10 +2,11 @@ package mill.util
 
 import scala.collection.mutable
 import Strict.Agg
+
 /**
   * A map from keys to collections of values: you can assign multiple values
   * to any particular key. Also allows lookups in both directions: what values
-  * are assigned to a key or what key a value is assigned ti.
+  * are assigned to a key or what key a value is assigned to.
   */
 trait MultiBiMap[K, V]{
   def containsValue(v: V): Boolean
@@ -22,6 +23,7 @@ trait MultiBiMap[K, V]{
 }
 
 object MultiBiMap{
+
   class Mutable[K, V]() extends MultiBiMap[K, V]{
     private[this] val valueToKey = mutable.LinkedHashMap.empty[V, K]
     private[this] val keyToValues = mutable.LinkedHashMap.empty[K, Agg.Mutable[V]]
diff --git a/main/src/mill/MillMain.scala b/main/src/mill/MillMain.scala
index 421b9f20..1598d5f3 100644
--- a/main/src/mill/MillMain.scala
+++ b/main/src/mill/MillMain.scala
@@ -64,8 +64,18 @@ object MillMain {
       }
     )
 
+    var debugLog = false
+    val debugLogSignature = Arg[Config, Unit](
+      name = "debug", shortName = Some('d'),
+      doc = "Show debug output on STDOUT",
+      (c, v) => {
+        debugLog = true
+        c
+      }
+    )
+
     val millArgSignature =
-      Cli.genericSignature.filter(a => !removed(a.name)) ++ Seq(interactiveSignature, disableTickerSignature)
+      Cli.genericSignature.filter(a => !removed(a.name)) ++ Seq(interactiveSignature, disableTickerSignature, debugLogSignature)
 
     Cli.groupArgs(
       args.toList,
@@ -105,7 +115,8 @@ object MillMain {
                   |  interp.colors(),
                   |  repl.pprinter(),
                   |  build.millSelf.get,
-                  |  build.millDiscover
+                  |  build.millDiscover,
+                  |  $debugLog
                   |)
                   |repl.pprinter() = replApplyHandler.pprinter
                   |import replApplyHandler.generatedEval._
@@ -120,7 +131,8 @@ object MillMain {
             stdout, stderr, stdin,
             stateCache,
             env,
-            setIdle
+            setIdle,
+            debugLog
           )
 
           if (mill.main.client.Util.isJava9OrAbove) {
diff --git a/main/src/mill/main/MainModule.scala b/main/src/mill/main/MainModule.scala
index 7e326860..929ad3dc 100644
--- a/main/src/mill/main/MainModule.scala
+++ b/main/src/mill/main/MainModule.scala
@@ -178,7 +178,7 @@ trait MainModule extends mill.Module{
         // When using `show`, redirect all stdout of the evaluated tasks so the
         // printed JSON is the only thing printed to stdout.
         log = evaluator.log match{
-          case PrintLogger(c1, d, c2, o, i, e, in) => PrintLogger(c1, d, c2, e, i, e, in)
+          case PrintLogger(c1, d, c2, o, i, e, in, de) => PrintLogger(c1, d, c2, e, i, e, in, de)
           case l => l
         }
       ),
diff --git a/main/src/mill/main/MainRunner.scala b/main/src/mill/main/MainRunner.scala
index 58c47998..4f31a724 100644
--- a/main/src/mill/main/MainRunner.scala
+++ b/main/src/mill/main/MainRunner.scala
@@ -24,7 +24,8 @@ class MainRunner(val config: ammonite.main.Cli.Config,
                  stdIn: InputStream,
                  stateCache0: Option[Evaluator.State] = None,
                  env : Map[String, String],
-                 setIdle: Boolean => Unit)
+                 setIdle: Boolean => Unit,
+                 debugLog: Boolean)
   extends ammonite.MainRunner(
     config, outprintStream, errPrintStream,
     stdIn, outprintStream, errPrintStream
@@ -80,7 +81,8 @@ class MainRunner(val config: ammonite.main.Cli.Config,
             outprintStream,
             errPrintStream,
             errPrintStream,
-            stdIn
+            stdIn,
+            debugEnabled = debugLog
           ),
           env
         )
diff --git a/main/src/mill/main/ReplApplyHandler.scala b/main/src/mill/main/ReplApplyHandler.scala
index 59a6780b..af69c761 100644
--- a/main/src/mill/main/ReplApplyHandler.scala
+++ b/main/src/mill/main/ReplApplyHandler.scala
@@ -16,7 +16,8 @@ object ReplApplyHandler{
                colors: ammonite.util.Colors,
                pprinter0: pprint.PPrinter,
                rootModule: mill.define.BaseModule,
-               discover: Discover[_]) = {
+               discover: Discover[_],
+               debugLog: Boolean) = {
     new ReplApplyHandler(
       pprinter0,
       new Evaluator(
@@ -31,7 +32,8 @@ object ReplApplyHandler{
           System.out,
           System.err,
           System.err,
-          System.in
+          System.in,
+          debugEnabled = debugLog
         )
       )
     )
diff --git a/main/src/mill/modules/Jvm.scala b/main/src/mill/modules/Jvm.scala
index f0c0d8a9..4c26b297 100644
--- a/main/src/mill/modules/Jvm.scala
+++ b/main/src/mill/modules/Jvm.scala
@@ -209,7 +209,22 @@ object Jvm {
     m
   }
 
-  def createJar(inputPaths: Agg[Path], mainClass: Option[String] = None)
+  /**
+    * Create a jar file containing all files from the specified input Paths,
+    * called out.jar in the implicit ctx.dest folder. An optional main class may
+    * be provided for the jar. An optional filter function may also be provided to
+    * selectively include/exclude specific files.
+    * @param inputPaths - `Agg` of `Path`s containing files to be included in the jar
+    * @param mainClass - optional main class for the jar
+    * @param fileFilter - optional file filter to select files to be included.
+    *                   Given a `Path` (from inputPaths) and a `RelPath` for the individual file,
+    *                   return true if the file is to be included in the jar.
+    * @param ctx - implicit `Ctx.Dest` used to determine the output directory for the jar.
+    * @return - a `PathRef` for the created jar.
+    */
+  def createJar(inputPaths: Agg[Path],
+                mainClass: Option[String] = None,
+                fileFilter: (Path, RelPath) => Boolean = (p: Path, r: RelPath) => true)
                (implicit ctx: Ctx.Dest): PathRef = {
     val outputPath = ctx.dest / "out.jar"
     rm(outputPath)
@@ -228,7 +243,7 @@ object Jvm {
         (file, mapping) <-
           if (p.isFile) Iterator(p -> empty/p.last)
           else ls.rec(p).filter(_.isFile).map(sub => sub -> sub.relativeTo(p))
-        if !seen(mapping)
+        if !seen(mapping) && fileFilter(p, mapping)
       } {
         seen.add(mapping)
         val entry = new JarEntry(mapping.toString)
diff --git a/main/test/src/mill/eval/JavaCompileJarTests.scala b/main/test/src/mill/eval/JavaCompileJarTests.scala
index 37e4c119..d4bdbd87 100644
--- a/main/test/src/mill/eval/JavaCompileJarTests.scala
+++ b/main/test/src/mill/eval/JavaCompileJarTests.scala
@@ -39,6 +39,8 @@ object JavaCompileJarTests extends TestSuite{
         def allSources = T{ sourceRoot().flatMap(p => ls.rec(p.path)).map(PathRef(_)) }
         def classFiles = T{ compileAll(allSources()) }
         def jar = T{ Jvm.createJar(Loose.Agg(classFiles().path) ++ resourceRoot().map(_.path)) }
+        // Test createJar() with optional file filter.
+        def filterJar(fileFilter: (Path, RelPath) => Boolean) = T{ Jvm.createJar(Loose.Agg(classFiles().path) ++ resourceRoot().map(_.path), None, fileFilter) }
 
         def run(mainClsName: String) = T.command{
           %%('java, "-Duser.language=en", "-cp", classFiles().path, mainClsName)
@@ -116,6 +118,13 @@ object JavaCompileJarTests extends TestSuite{
           |""".stripMargin
       assert(jarContents.lines.toSeq == expectedJarContents.lines.toSeq)
 
+      // Create the Jar again, but this time, filter out the Foo files.
+      def noFoos(s: String) = !s.contains("Foo")
+      val filterFunc = (p: Path, r: RelPath) => noFoos(r.last)
+      eval(filterJar(filterFunc))
+      val filteredJarContents = %%('jar, "-tf", evaluator.outPath/'filterJar/'dest/"out.jar")(evaluator.outPath).out.string
+      assert(filteredJarContents.lines.toSeq == expectedJarContents.lines.filter(noFoos(_)).toSeq)
+
       val executed = %%('java, "-cp", evaluator.outPath/'jar/'dest/"out.jar", "test.Foo")(evaluator.outPath).out.string
       assert(executed == (31337 + 271828) + System.lineSeparator)
 
diff --git a/main/test/src/mill/util/ScriptTestSuite.scala b/main/test/src/mill/util/ScriptTestSuite.scala
index 0b0c4011..9d3edb65 100644
--- a/main/test/src/mill/util/ScriptTestSuite.scala
+++ b/main/test/src/mill/util/ScriptTestSuite.scala
@@ -15,10 +15,11 @@ abstract class ScriptTestSuite(fork: Boolean) extends TestSuite{
   val stdOutErr = new PrintStream(new ByteArrayOutputStream())
   val stdIn = new ByteArrayInputStream(Array())
   val disableTicker = false
+  val debugLog = false
   lazy val runner = new mill.main.MainRunner(
     ammonite.main.Cli.Config(wd = wd), disableTicker,
     stdOutErr, stdOutErr, stdIn, None, Map.empty,
-    b => ()
+    b => (), debugLog
   )
   def eval(s: String*) = {
     if (!fork) runner.runScript(workspacePath / buildPath , s.toList)
diff --git a/main/test/src/mill/util/TestEvaluator.scala b/main/test/src/mill/util/TestEvaluator.scala
index 1a114947..6e7fe484 100644
--- a/main/test/src/mill/util/TestEvaluator.scala
+++ b/main/test/src/mill/util/TestEvaluator.scala
@@ -26,7 +26,7 @@ class TestEvaluator(module: TestUtil.BaseModule)
 //  val logger = DummyLogger
   val logger = new PrintLogger(
     colored = true, disableTicker=false,
-    ammonite.util.Colors.Default, System.out, System.out, System.err, System.in
+    ammonite.util.Colors.Default, System.out, System.out, System.err, System.in, debugEnabled = false
  )
   val evaluator = new Evaluator(Ctx.defaultHome, outPath, TestEvaluator.externalOutPath, module, logger)
 
diff --git a/readme.md b/readme.md
index 634613bf..4233b7fc 100644
--- a/readme.md
+++ b/readme.md
@@ -151,6 +151,13 @@ optimizer without classpath conflicts.
 
 ## Changelog
 
+### {master}
+
+- Added new `debug` method to context logger, to log additional debug info into the
+  task specific output dir (`out/<task>/log`)
+
+- Added `--debug` option to enable debug output to STDERR
+
 ### 0.3.2
 
 - Automatically detect main class to make `ScalaModule#assembly` self-executable
diff --git a/scalalib/src/mill/scalalib/TestRunner.scala b/scalalib/src/mill/scalalib/TestRunner.scala
index 947021ba..0b9c897f 100644
--- a/scalalib/src/mill/scalalib/TestRunner.scala
+++ b/scalalib/src/mill/scalalib/TestRunner.scala
@@ -37,7 +37,8 @@ object TestRunner {
           System.out,
           System.err,
           System.err,
-          System.in
+          System.in,
+          debugEnabled = false
         )
         val home = Path(homeStr)
       }