Add support for external modules, useful for things `GenIdea` support

15 files changed, 136 insertions, 62 deletions

diff --git a/build.sc b/build.sc
index aa59e0f5..97b9afb8 100755
--- a/build.sc
+++ b/build.sc
@@ -253,4 +253,3 @@ def releaseCI(githubAuthKey: String, sonatypeCreds: String, gpgPassphrase: Strin
     ()
   }
 
-def idea() = T.command{ mill.scalalib.GenIdea() }
diff --git a/core/src/mill/define/BaseModule.scala b/core/src/mill/define/BaseModule.scala
new file mode 100644
index 00000000..9cefeeae
--- /dev/null
+++ b/core/src/mill/define/BaseModule.scala
@@ -0,0 +1,47 @@
+package mill.define
+
+import ammonite.main.Router.Overrides
+import ammonite.ops.Path
+
+object BaseModule{
+  case class Implicit(value: BaseModule)
+}
+class BaseModule(millSourcePath0: Path, external0: Boolean = false)
+                (implicit millModuleEnclosing0: sourcecode.Enclosing,
+                 millModuleLine0: sourcecode.Line,
+                 millName0: sourcecode.Name)
+  extends Module()(
+    mill.define.Ctx.make(
+      implicitly,
+      implicitly,
+      implicitly,
+      BasePath(millSourcePath0),
+      Segments(),
+      Overrides(0),
+      Ctx.External(external0)
+    )
+  ){
+  // A BaseModule should provide an empty Segments list to it's children, since
+  // it is the root of the module tree, and thus must not include it's own
+  // sourcecode.Name as part of the list,
+  override implicit def millModuleSegments: Segments = Segments()
+  override def millSourcePath = millOuterCtx.millSourcePath
+  override implicit def millModuleBasePath: BasePath = BasePath(millSourcePath)
+  implicit def millImplicitBaseModule: BaseModule.Implicit = BaseModule.Implicit(this)
+}
+
+
+abstract class ExternalModule(implicit millModuleEnclosing0: sourcecode.Enclosing,
+                              millModuleLine0: sourcecode.Line,
+                              millName0: sourcecode.Name)
+  extends BaseModule(ammonite.ops.pwd, external0 = true){
+  def millDiscover: Discover[_]
+  implicit def millDiscoverImplicit: Discover[_] = millDiscover
+  assert(
+    !" #".exists(millModuleEnclosing0.value.contains(_)),
+    "External modules must be at a top-level static path, not " + millModuleEnclosing0.value
+  )
+  override implicit def millModuleSegments = {
+    Segments(millModuleEnclosing0.value.split('.').map(Segment.Label):_*)
+  }
+}
\ No newline at end of file
diff --git a/core/src/mill/define/Discover.scala b/core/src/mill/define/Discover.scala
index ab2ba900..98e29a76 100644
--- a/core/src/mill/define/Discover.scala
+++ b/core/src/mill/define/Discover.scala
@@ -53,11 +53,11 @@ object Discover {
         } yield (m.overrides.length, router.extractMethod(m, curCls).asInstanceOf[c.Tree])
       }
       if overridesRoutes.nonEmpty
-      val (overrides, routes) = overridesRoutes.unzip
-
     } yield {
+      val (overrides, routes) = overridesRoutes.unzip
       val lhs =  q"classOf[${discoveredModuleType.typeSymbol.asClass}]"
-      val rhs = q"scala.Seq[(Int, ammonite.main.Router.EntryPoint[${discoveredModuleType.typeSymbol.asClass}])](..$overridesRoutes)"
+      val clsType = discoveredModuleType.typeSymbol.asClass
+      val rhs = q"scala.Seq[(Int, ammonite.main.Router.EntryPoint[$clsType])](..$overridesRoutes)"
       q"$lhs -> $rhs"
     }
 
diff --git a/core/src/mill/define/Module.scala b/core/src/mill/define/Module.scala
index e765a6b4..66132fa9 100644
--- a/core/src/mill/define/Module.scala
+++ b/core/src/mill/define/Module.scala
@@ -90,42 +90,3 @@ object Module{
 trait TaskModule extends Module {
   def defaultCommandName(): String
 }
-object BaseModule{
-  case class Implicit(value: BaseModule)
-}
-class BaseModule(millSourcePath0: Path, external0: Boolean = false)
-                (implicit millModuleEnclosing0: sourcecode.Enclosing,
-                 millModuleLine0: sourcecode.Line,
-                 millName0: sourcecode.Name)
-  extends Module()(
-    mill.define.Ctx.make(
-      implicitly,
-      implicitly,
-      implicitly,
-      BasePath(millSourcePath0),
-      Segments(),
-      Overrides(0),
-      Ctx.External(external0)
-    )
-  ){
-  // A BaseModule should provide an empty Segments list to it's children, since
-  // it is the root of the module tree, and thus must not include it's own
-  // sourcecode.Name as part of the list,
-  override implicit def millModuleSegments: Segments = Segments()
-  override def millSourcePath = millOuterCtx.millSourcePath
-  override implicit def millModuleBasePath: BasePath = BasePath(millSourcePath)
-  implicit def millImplicitBaseModule: BaseModule.Implicit = BaseModule.Implicit(this)
-}
-
-class ExternalModule(implicit millModuleEnclosing0: sourcecode.Enclosing,
-                     millModuleLine0: sourcecode.Line,
-                     millName0: sourcecode.Name)
-  extends BaseModule(ammonite.ops.pwd, external0 = true){
-  assert(
-    !" #".exists(millModuleEnclosing0.value.contains(_)),
-    "External modules must be at a top-level static path, not " + millModuleEnclosing0.value
-  )
-  override implicit def millModuleSegments = {
-    Segments(millModuleEnclosing0.value.split('.').map(Segment.Label):_*)
-  }
-}
\ No newline at end of file
diff --git a/core/src/mill/eval/Evaluator.scala b/core/src/mill/eval/Evaluator.scala
index bb78a020..3d6d253b 100644
--- a/core/src/mill/eval/Evaluator.scala
+++ b/core/src/mill/eval/Evaluator.scala
@@ -58,7 +58,13 @@ class Evaluator[T](val outPath: Path,
                   }
               }
             }
-            findMatching(c.cls).get.find(_._2.name == c.ctx.segment.pathSegments.head).get._1
+
+            findMatching(c.cls) match{
+              case Some(v) => v.find(_._2.name == c.ctx.segment.pathSegments.head).get._1
+              // For now we don't properly support overrides for external modules
+              // that do not appear in the Evaluator's main Discovered listing
+              case None => 0
+            }
 
           case c: mill.define.Worker[_] => 0
         }
diff --git a/core/src/mill/main/ParseArgs.scala b/core/src/mill/main/ParseArgs.scala
index 7170cc60..bafcd907 100644
--- a/core/src/mill/main/ParseArgs.scala
+++ b/core/src/mill/main/ParseArgs.scala
@@ -2,12 +2,12 @@ package mill.main
 
 import mill.util.EitherOps
 import fastparse.all._
-import mill.define.Segment
+import mill.define.{Segment, Segments}
 
 object ParseArgs {
 
   def apply(scriptArgs: Seq[String])
-    : Either[String, (List[List[Segment]], Seq[String])] = {
+    : Either[String, (List[(Option[Segments], Segments)], Seq[String])] = {
     val (selectors, args, isMultiSelectors) = extractSelsAndArgs(scriptArgs)
     for {
       _ <- validateSelectors(selectors)
@@ -119,7 +119,7 @@ object ParseArgs {
       .parse(input)
   }
 
-  def extractSegments(selectorString: String): Either[String, List[Segment]] =
+  def extractSegments(selectorString: String): Either[String, (Option[Segments], Segments)] =
     parseSelector(selectorString) match {
       case f: Parsed.Failure           => Left(s"Parsing exception ${f.msg}")
       case Parsed.Success(selector, _) => Right(selector)
@@ -131,8 +131,12 @@ object ParseArgs {
     val ident2 = P( CharsWhileIn(identChars ++ ".") ).!
     val segment = P( ident ).map( Segment.Label)
     val crossSegment = P("[" ~ ident2.rep(1, sep = ",") ~ "]").map(Segment.Cross)
-    val query = P(segment ~ ("." ~ segment | crossSegment).rep ~ End).map {
-      case (h, rest) => h :: rest.toList
+    val simpleQuery = P(segment ~ ("." ~ segment | crossSegment).rep).map {
+      case (h, rest) => Segments(h :: rest.toList:_*)
+    }
+    val query = P( simpleQuery ~ ("/" ~/ simpleQuery).?).map{
+      case (q, None) => (None, q)
+      case (q, Some(q2)) => (Some(q), q2)
     }
     query.parse(input)
   }
diff --git a/core/src/mill/main/RunScript.scala b/core/src/mill/main/RunScript.scala
index a05c7623..2efe0e97 100644
--- a/core/src/mill/main/RunScript.scala
+++ b/core/src/mill/main/RunScript.scala
@@ -8,7 +8,7 @@ import ammonite.runtime.SpecialClassLoader
 import ammonite.util.Util.CodeSource
 import ammonite.util.{Name, Res, Util}
 import mill.{PathRef, define}
-import mill.define.{Discover, Segment, Task}
+import mill.define.{Discover, ExternalModule, Segment, Task}
 import mill.eval.{Evaluator, Result}
 import mill.util.{EitherOps, Logger}
 import mill.util.Strict.Agg
@@ -135,15 +135,26 @@ object RunScript{
       parsed <- ParseArgs(scriptArgs)
       (selectors, args) = parsed
       targets <- {
-        val selected = selectors.map { sel =>
-          val crossSelectors = sel.map {
+        val selected = selectors.map { case (scopedSel, sel) =>
+          val (rootModule, discover) = scopedSel match{
+            case None => (evaluator.rootModule, evaluator.discover)
+            case Some(scoping) =>
+              val moduleCls =
+                evaluator.rootModule.getClass.getClassLoader.loadClass(scoping.render + "$")
+
+              pprint.log(moduleCls.getFields)
+              pprint.log(moduleCls.getMethods)
+              val rootModule = moduleCls.getField("MODULE$").get(moduleCls).asInstanceOf[ExternalModule]
+              (rootModule, rootModule.millDiscover)
+          }
+          val crossSelectors = sel.value.map {
             case Segment.Cross(x) => x.toList.map(_.toString)
             case _ => Nil
           }
           mill.main.Resolve.resolve(
-            sel, evaluator.rootModule,
-            evaluator.discover,
-            args, crossSelectors, Nil
+            sel.value.toList, rootModule,
+            discover,
+            args, crossSelectors.toList, Nil
           )
         }
         EitherOps.sequence(selected)
diff --git a/core/test/src/mill/eval/ModuleTests.scala b/core/test/src/mill/eval/ModuleTests.scala
index b452a854..c6061abb 100644
--- a/core/test/src/mill/eval/ModuleTests.scala
+++ b/core/test/src/mill/eval/ModuleTests.scala
@@ -3,6 +3,7 @@ package mill.eval
 import ammonite.ops._
 import mill.util.{TestEvaluator, TestUtil}
 import mill.T
+import mill.define.Discover
 import mill.util.TestEvaluator.implicitDisover
 import utest._
 
@@ -13,6 +14,7 @@ object ModuleTests extends TestSuite{
     object inner extends mill.Module{
       def y = T{17}
     }
+    def millDiscover = Discover[this.type]
   }
   object Build extends TestUtil.BaseModule{
     def z = T{ ExternalModule.x() + ExternalModule.inner.y() }
@@ -35,6 +37,7 @@ object ModuleTests extends TestSuite{
       object Build extends mill.define.ExternalModule {
 
         def z = T{ ExternalModule.x() + ExternalModule.inner.y() }
+        def millDiscover = Discover[this.type]
       }
 
       intercept[java.lang.AssertionError]{ Build }
diff --git a/core/test/src/mill/main/MainTests.scala b/core/test/src/mill/main/MainTests.scala
index 3d8e4d5c..22f93ae0 100644
--- a/core/test/src/mill/main/MainTests.scala
+++ b/core/test/src/mill/main/MainTests.scala
@@ -14,8 +14,10 @@ object MainTests extends TestSuite{
     val expected = expected0.map(_.map(_(module)))
     val resolved = for{
       selectors <- mill.main.ParseArgs(Seq(selectorString)).map(_._1.head)
-      val crossSelectors = selectors.map{case Segment.Cross(x) => x.toList.map(_.toString) case _ => Nil}
-      task <- mill.main.Resolve.resolve(selectors, module, discover, Nil, crossSelectors, Nil)
+      val crossSelectors = selectors._2.value.map{case Segment.Cross(x) => x.toList.map(_.toString) case _ => Nil}
+      task <- mill.main.Resolve.resolve(
+        selectors._2.value.toList, module, discover, Nil, crossSelectors.toList, Nil
+      )
     } yield task
     assert(resolved == expected)
   }
diff --git a/core/test/src/mill/util/ScriptTestSuite.scala b/core/test/src/mill/util/ScriptTestSuite.scala
index 7a963ef9..1aa74de1 100644
--- a/core/test/src/mill/util/ScriptTestSuite.scala
+++ b/core/test/src/mill/util/ScriptTestSuite.scala
@@ -22,7 +22,7 @@ abstract class ScriptTestSuite extends TestSuite{
   def meta(s: String) = {
     val (List(selector), args) = ParseArgs.apply(Seq(s)).right.get
 
-    read(workspacePath / "out" / selector.flatMap(_.pathSegments) / "meta.json")
+    read(workspacePath / "out" / selector._2.value.flatMap(_.pathSegments) / "meta.json")
   }
 
 
diff --git a/docs/modules.md b/docs/modules.md
index b92ab684..8bbe6d49 100644
--- a/docs/modules.md
+++ b/docs/modules.md
@@ -122,4 +122,37 @@ would have it's `millSourcePath` be `./foo/lols/baz`.
 Note that `millSourcePath` is generally only used for a module's input source files.
 Output is always in the `out/` folder and cannot be changed, e.g. even with the
 overriden `millSourcePath` the output paths are still the default `./out/foo/bar` and
-`./out/foo/baz/qux` folders.
\ No newline at end of file
+`./out/foo/baz/qux` folders.
+
+## External Modules
+
+Libraries for use in Mill can define `ExternalModule`s: `Module`s which are
+shared between all builds which use that library:
+
+```scala
+package foo
+import mill._
+
+object Bar extends mill.define.ExternalModule {
+  def baz = T{ 1 }
+  def qux() = T.command{ println(baz() + 1) }
+
+  def millDiscover = mill.define.Discover[this.type]
+}
+```
+
+In the above example, `foo.Bar` is an `ExternalModule` living within the `foo`
+Java package, containing the `baz` target and `qux` command. Those can be run
+from the command line via:
+
+```bash
+mill foo.Bar/baz
+mill foo.Bar/qux
+```
+
+`ExternalModule`s are useful for someone providing a library for use with Mill
+that is shared by the entire build: for example,
+`mill.scalalib.ScalaWorkerApi/scalaWorker` provides a shared Scala compilation
+service & cache that is shared between all `ScalaModule`s, and
+`mill.scalalib.GenIdeaModule/idea` lets you generate IntelliJ projects without
+needing to define your own `T.command` in your `build.sc` file
\ No newline at end of file
diff --git a/readme.md b/readme.md
index 5552e402..4dc6ca1c 100644
--- a/readme.md
+++ b/readme.md
@@ -65,7 +65,7 @@ sbt "~bin/test:run"
 Lastly, you can generate IntelliJ Scala project files using Mill via
 
 ```bash
-./target/bin/mill idea
+./target/bin/mill mill.scalalib.GenIdeaModule/idea
 ```
 
 Allowing you to import a Mill project into Intellij without using SBT
diff --git a/scalajslib/src/mill/scalajslib/ScalaJSBridge.scala b/scalajslib/src/mill/scalajslib/ScalaJSBridge.scala
index 8152d3d1..780fa09e 100644
--- a/scalajslib/src/mill/scalajslib/ScalaJSBridge.scala
+++ b/scalajslib/src/mill/scalajslib/ScalaJSBridge.scala
@@ -4,6 +4,7 @@ import java.io.File
 import java.net.URLClassLoader
 
 import ammonite.ops.Path
+import mill.define.Discover
 import mill.{Agg, T}
 
 sealed trait OptimizeMode
@@ -72,4 +73,5 @@ trait ScalaJSBridge {
 object ScalaJSBridge extends mill.define.ExternalModule {
 
   def scalaJSBridge = T.worker { new ScalaJSWorker() }
+  def millDiscover = Discover[this.type]
 }
diff --git a/scalalib/src/mill/scalalib/GenIdea.scala b/scalalib/src/mill/scalalib/GenIdea.scala
index f4d90068..2f76b666 100644
--- a/scalalib/src/mill/scalalib/GenIdea.scala
+++ b/scalalib/src/mill/scalalib/GenIdea.scala
@@ -5,11 +5,16 @@ import coursier.Cache
 import coursier.maven.MavenRepository
 import mill.define._
 import mill.eval.{Evaluator, PathRef, Result}
-import mill.scalalib
+import mill.{T, scalalib}
 import mill.util.Ctx.Log
 import mill.util.{Loose, PrintLogger, Strict}
 import mill.util.Strict.Agg
 
+
+object GenIdeaModule extends ExternalModule {
+  def idea() = T.command{ mill.scalalib.GenIdea() }
+  def millDiscover = Discover[this.type]
+}
 object GenIdea {
 
   def apply()(implicit ctx: Log,
diff --git a/scalalib/src/mill/scalalib/ScalaWorkerApi.scala b/scalalib/src/mill/scalalib/ScalaWorkerApi.scala
index 0a9e8f5d..e66864af 100644
--- a/scalalib/src/mill/scalalib/ScalaWorkerApi.scala
+++ b/scalalib/src/mill/scalalib/ScalaWorkerApi.scala
@@ -7,7 +7,7 @@ import coursier.maven.MavenRepository
 import mill.Agg
 import mill.scalalib.TestRunner.Result
 import mill.T
-import mill.define.Worker
+import mill.define.{Discover, Worker}
 import mill.scalalib.Lib.resolveDependencies
 import mill.util.Loose
 import mill.util.JsonFormatters._
@@ -42,6 +42,7 @@ object ScalaWorkerApi extends mill.define.ExternalModule {
       Seq(ivy"org.scala-sbt:compiler-interface:1.1.0")
     )
   }
+  def millDiscover = Discover[this.type]
 }
 
 trait ScalaWorkerApi {