ability to run multiple tasks via bash/zsh braces expansion (#104) fixes #31

5 files changed, 415 insertions, 49 deletions

diff --git a/core/src/main/scala/mill/main/ParseArgs.scala b/core/src/main/scala/mill/main/ParseArgs.scala
new file mode 100644
index 00000000..dc848418
--- /dev/null
+++ b/core/src/main/scala/mill/main/ParseArgs.scala
@@ -0,0 +1,142 @@
+package mill.main
+
+import mill.util.EitherOps
+import fastparse.all._
+import mill.define.Segment
+
+object ParseArgs {
+
+  def apply(scriptArgs: Seq[String])
+    : Either[String, (List[List[Segment]], Seq[String])] = {
+    val (selectors, args, isMultiSelectors) = extractSelsAndArgs(scriptArgs)
+    for {
+      _ <- validateSelectors(selectors)
+      expandedSelectors <- EitherOps
+        .sequence(selectors.map(expandBraces))
+        .map(_.flatten)
+      _ <- validateExpanded(expandedSelectors, isMultiSelectors)
+      selectors <- EitherOps.sequence(expandedSelectors.map(extractSegments))
+    } yield (selectors.toList, args)
+  }
+
+  def extractSelsAndArgs(
+      scriptArgs: Seq[String]): (Seq[String], Seq[String], Boolean) = {
+    val multiFlags = Seq("--all", "--seq")
+    val isMultiSelectors = scriptArgs.headOption.exists(multiFlags.contains)
+
+    if (isMultiSelectors) {
+      val dd = scriptArgs.indexOf("--")
+      val selectors = (if (dd == -1) scriptArgs
+                       else scriptArgs.take(dd)).filterNot(multiFlags.contains)
+      val args = if (dd == -1) Seq.empty else scriptArgs.drop(dd + 1)
+
+      (selectors, args, isMultiSelectors)
+    } else {
+      (scriptArgs.take(1), scriptArgs.drop(1), isMultiSelectors)
+    }
+  }
+
+  private def validateSelectors(
+      selectors: Seq[String]): Either[String, Unit] = {
+    if (selectors.isEmpty || selectors.exists(_.isEmpty))
+      Left("Selector cannot be empty")
+    else Right(())
+  }
+
+  private def validateExpanded(expanded: Seq[String],
+                               isMulti: Boolean): Either[String, Unit] = {
+    if (!isMulti && expanded.length > 1)
+      Left("Please use --all flag to run multiple tasks")
+    else Right(())
+  }
+
+  def expandBraces(selectorString: String): Either[String, List[String]] = {
+    parseBraceExpansion(selectorString) match {
+      case f: Parsed.Failure           => Left(s"Parsing exception ${f.msg}")
+      case Parsed.Success(expanded, _) => Right(expanded.toList)
+    }
+  }
+
+  private sealed trait Fragment
+  private object Fragment {
+    case class Keep(value: String) extends Fragment
+    case class Expand(values: List[List[Fragment]]) extends Fragment
+
+    def unfold(fragments: List[Fragment]): Seq[String] = {
+      fragments match {
+        case head :: rest =>
+          val prefixes = head match {
+            case Keep(v)          => Seq(v)
+            case Expand(Nil)      => Seq("{}")
+            case Expand(List(vs)) => unfold(vs).map("{" + _ + "}")
+            case Expand(vss)      => vss.flatMap(unfold)
+          }
+          for {
+            prefix <- prefixes
+            suffix <- unfold(rest)
+          } yield prefix + suffix
+
+        case Nil => Seq("")
+      }
+    }
+  }
+
+  private object BraceExpansionParser {
+    val plainChars =
+      P(CharsWhile(c => c != ',' && c != '{' && c != '}')).!.map(Fragment.Keep)
+
+    val toExpand: P[Fragment] =
+      P("{" ~ braceParser.rep(1).rep(sep = ",") ~ "}").map(
+        x => Fragment.Expand(x.toList.map(_.toList))
+      )
+
+    val braceParser = P(toExpand | plainChars)
+
+    val parser = P(braceParser.rep(1).rep(sep = ",") ~ End)
+  }
+
+  private def parseBraceExpansion(input: String) = {
+    def unfold(vss: List[Seq[String]]): Seq[String] = {
+      vss match {
+        case Nil => Seq("")
+        case head :: rest =>
+          for {
+            str <- head
+            r <- unfold(rest)
+          } yield
+            r match {
+              case "" => str
+              case _  => str + "," + r
+            }
+      }
+    }
+
+    BraceExpansionParser.parser
+      .map { vss =>
+        val stringss = vss.map(x => Fragment.unfold(x.toList)).toList
+        unfold(stringss)
+      }
+      .parse(input)
+  }
+
+  def extractSegments(selectorString: String): Either[String, List[Segment]] =
+    parseSelector(selectorString) match {
+      case f: Parsed.Failure           => Left(s"Parsing exception ${f.msg}")
+      case Parsed.Success(selector, _) => Right(selector)
+    }
+
+  private def parseSelector(input: String) = {
+    val segment =
+      P(CharsWhileIn(('a' to 'z') ++ ('A' to 'Z') ++ ('0' to '9')).!).map(
+        Segment.Label
+      )
+    val crossSegment =
+      P("[" ~ CharsWhile(c => c != ',' && c != ']').!.rep(1, sep = ",") ~ "]")
+        .map(Segment.Cross)
+    val query = P(segment ~ ("." ~ segment | crossSegment).rep ~ End).map {
+      case (h, rest) => h :: rest.toList
+    }
+    query.parse(input)
+  }
+
+}
diff --git a/core/src/main/scala/mill/main/RunScript.scala b/core/src/main/scala/mill/main/RunScript.scala
index e79349ad..b8793b40 100644
--- a/core/src/main/scala/mill/main/RunScript.scala
+++ b/core/src/main/scala/mill/main/RunScript.scala
@@ -1,6 +1,5 @@
 package mill.main
 
-import java.io.{ByteArrayOutputStream, PrintStream}
 import java.nio.file.NoSuchFileException
 
 import ammonite.interp.Interpreter
@@ -10,9 +9,9 @@ import ammonite.util.{Name, Res, Util}
 import mill.{PathRef, define}
 import mill.define.Task
 import mill.define.Segment
-import mill.discover.{Discovered, Mirror}
-import mill.eval.{Evaluator, PathRef, Result}
-import mill.util.{Logger, OSet, PrintLogger}
+import mill.discover.Discovered
+import mill.eval.{Evaluator, Result}
+import mill.util.{EitherOps, Logger, OSet}
 import upickle.Js
 
 /**
@@ -113,28 +112,32 @@ object RunScript{
       _ <- Res(consistencyCheck(mapping))
     } yield mapping
   }
+
   def evaluateTarget[T](evaluator: Evaluator[_],
                         scriptArgs: Seq[String]) = {
-
-    val selectorString = scriptArgs.headOption.getOrElse("")
-    val rest = scriptArgs.drop(1)
-
     for {
-      sel <- parseArgs(selectorString)
-      crossSelectors = sel.map{
-        case Segment.Cross(x) => x.toList.map(_.toString)
-        case _ => Nil
+      parsed <- ParseArgs(scriptArgs)
+      (selectors, args) = parsed
+      targets <- {
+        val selected = selectors.map { sel =>
+          val crossSelectors = sel.map {
+            case Segment.Cross(x) => x.toList.map(_.toString)
+            case _ => Nil
+          }
+          mill.main.Resolve.resolve(
+            sel, evaluator.mapping.mirror, evaluator.mapping.base,
+            args, crossSelectors, Nil
+          )
+        }
+        EitherOps.sequence(selected)
       }
-      target <- mill.main.Resolve.resolve(
-        sel, evaluator.mapping.mirror, evaluator.mapping.base,
-        rest, crossSelectors, Nil
-      )
-      (watched, res) = evaluate(evaluator, target)
+      (watched, res) = evaluate(evaluator, targets)
     } yield (watched, res)
   }
+
   def evaluate(evaluator: Evaluator[_],
-               target: Task[Any]): (Seq[PathRef], Either[String, Seq[(Any, Option[upickle.Js.Value])]]) = {
-    val evaluated = evaluator.evaluate(OSet(target))
+               targets: Seq[Task[Any]]): (Seq[PathRef], Either[String, Seq[(Any, Option[upickle.Js.Value])]]) = {
+    val evaluated = evaluator.evaluate(OSet.from(targets))
     val watched = evaluated.results
       .iterator
       .collect {
@@ -158,7 +161,7 @@ object RunScript{
 
     evaluated.failing.keyCount match {
       case 0 =>
-        val json = for(t <- Seq(target)) yield {
+        val json = for(t <- targets) yield {
           t match {
             case t: mill.define.NamedTask[_] =>
               val jsonFile = Evaluator
@@ -176,32 +179,6 @@ object RunScript{
     }
   }
 
-  def parseSelector(input: String) = {
-    import fastparse.all._
-    val segment = P( CharsWhileIn(('a' to 'z') ++ ('A' to 'Z') ++ ('0' to '9')).! ).map(
-      Segment.Label
-    )
-    val crossSegment = P( "[" ~ CharsWhile(c => c != ',' && c != ']').!.rep(1, sep=",") ~ "]" ).map(
-      Segment.Cross
-    )
-    val query = P( segment ~ ("." ~ segment | crossSegment).rep ~ End ).map{
-      case (h, rest) => h :: rest.toList
-    }
-    query.parse(input)
-  }
-
-
-
-  def parseArgs(selectorString: String): Either[String, List[Segment]] = {
-    import fastparse.all.Parsed
-    if (selectorString.isEmpty) Left("Selector cannot be empty")
-    else parseSelector(selectorString) match {
-      case f: Parsed.Failure => Left(s"Parsing exception ${f.msg}")
-      case Parsed.Success(selector, _) => Right(selector)
-    }
-  }
-
-
   def consistencyCheck[T](mapping: Discovered.Mapping[T]): Either[String, Unit] = {
     val consistencyErrors = Discovered.consistencyCheck(mapping)
     if (consistencyErrors.nonEmpty) {
diff --git a/core/src/main/scala/mill/util/EitherOps.scala b/core/src/main/scala/mill/util/EitherOps.scala
new file mode 100644
index 00000000..da2552c8
--- /dev/null
+++ b/core/src/main/scala/mill/util/EitherOps.scala
@@ -0,0 +1,18 @@
+package mill.util
+
+import scala.collection.generic.CanBuildFrom
+import scala.collection.mutable
+import scala.language.higherKinds
+
+object EitherOps {
+
+  // implementation similar to scala.concurrent.Future#sequence
+  def sequence[A, B, M[X] <: TraversableOnce[X]](in: M[Either[A, B]])(
+      implicit cbf: CanBuildFrom[M[Either[A, B]], B, M[B]]): Either[A, M[B]] = {
+    in.foldLeft[Either[A, mutable.Builder[B, M[B]]]](Right(cbf(in))) {
+        case (acc, el) =>
+          for (a <- acc; e <- el) yield a += e
+      }
+      .map(_.result())
+  }
+}
diff --git a/core/src/test/scala/mill/main/MainTests.scala b/core/src/test/scala/mill/main/MainTests.scala
index 013729ff..b4568267 100644
--- a/core/src/test/scala/mill/main/MainTests.scala
+++ b/core/src/test/scala/mill/main/MainTests.scala
@@ -13,9 +13,9 @@ object MainTests extends TestSuite{
                expected: Either[String, Task[_]]) = {
 
     val resolved = for{
-      args <- mill.main.RunScript.parseArgs(selectorString)
-      val crossSelectors = args.map{case Segment.Cross(x) => x.toList.map(_.toString) case _ => Nil}
-      task <- mill.main.Resolve.resolve(args, mapping.mirror, mapping.base, Nil, crossSelectors, Nil)
+      selectors <- mill.main.ParseArgs(Seq(selectorString)).map(_._1.head)
+      val crossSelectors = selectors.map{case Segment.Cross(x) => x.toList.map(_.toString) case _ => Nil}
+      task <- mill.main.Resolve.resolve(selectors, mapping.mirror, mapping.base, Nil, crossSelectors, Nil)
     } yield task
     assert(resolved == expected)
   }
diff --git a/core/src/test/scala/mill/main/ParseArgsTest.scala b/core/src/test/scala/mill/main/ParseArgsTest.scala
new file mode 100644
index 00000000..2ef07d36
--- /dev/null
+++ b/core/src/test/scala/mill/main/ParseArgsTest.scala
@@ -0,0 +1,229 @@
+package mill.main
+
+import mill.define.Segment
+import mill.define.Segment.{Cross, Label}
+import utest._
+
+object ParseArgsTest extends TestSuite {
+
+  val tests = Tests {
+    'extractSelsAndArgs - {
+      def check(input: Seq[String],
+                expectedSelectors: Seq[String],
+                expectedArgs: Seq[String],
+                expectedIsMulti: Boolean) = {
+        val (selectors, args, isMulti) = ParseArgs.extractSelsAndArgs(input)
+
+        assert(
+          selectors == expectedSelectors,
+          args == expectedArgs,
+          isMulti == expectedIsMulti
+        )
+      }
+
+      'empty - check(input = Seq.empty,
+                     expectedSelectors = Seq.empty,
+                     expectedArgs = Seq.empty,
+                     expectedIsMulti = false)
+      'singleSelector - check(
+        input = Seq("core.compile"),
+        expectedSelectors = Seq("core.compile"),
+        expectedArgs = Seq.empty,
+        expectedIsMulti = false
+      )
+      'singleSelectorWithArgs - check(
+        input = Seq("application.run", "hello", "world"),
+        expectedSelectors = Seq("application.run"),
+        expectedArgs = Seq("hello", "world"),
+        expectedIsMulti = false
+      )
+      'singleSelectorWithAllInArgs - check(
+        input = Seq("application.run", "hello", "world", "--all"),
+        expectedSelectors = Seq("application.run"),
+        expectedArgs = Seq("hello", "world", "--all"),
+        expectedIsMulti = false
+      )
+      'multiSelectors - check(
+        input = Seq("--all", "core.jar", "core.docsJar", "core.sourcesJar"),
+        expectedSelectors = Seq("core.jar", "core.docsJar", "core.sourcesJar"),
+        expectedArgs = Seq.empty,
+        expectedIsMulti = true
+      )
+      'multiSelectorsSeq - check(
+        input = Seq("--seq", "core.jar", "core.docsJar", "core.sourcesJar"),
+        expectedSelectors = Seq("core.jar", "core.docsJar", "core.sourcesJar"),
+        expectedArgs = Seq.empty,
+        expectedIsMulti = true
+      )
+      'multiSelectorsWithArgs - check(
+        input = Seq("--all",
+                    "core.compile",
+                    "application.runMain",
+                    "--",
+                    "Main",
+                    "hello",
+                    "world"),
+        expectedSelectors = Seq("core.compile", "application.runMain"),
+        expectedArgs = Seq("Main", "hello", "world"),
+        expectedIsMulti = true
+      )
+      'multiSelectorsWithArgsWithAllInArgs - check(
+        input = Seq("--all",
+                    "core.compile",
+                    "application.runMain",
+                    "--",
+                    "Main",
+                    "--all",
+                    "world"),
+        expectedSelectors = Seq("core.compile", "application.runMain"),
+        expectedArgs = Seq("Main", "--all", "world"),
+        expectedIsMulti = true
+      )
+    }
+    'expandBraces - {
+      def check(input: String, expectedExpansion: List[String]) = {
+        val Right(expanded) = ParseArgs.expandBraces(input)
+
+        assert(expanded == expectedExpansion)
+      }
+
+      'expandLeft - check(
+        "{application,core}.compile",
+        List("application.compile", "core.compile")
+      )
+      'expandRight - check(
+        "application.{jar,docsJar,sourcesJar}",
+        List("application.jar", "application.docsJar", "application.sourcesJar")
+      )
+      'expandBoth - check(
+        "{core,application}.{jar,docsJar}",
+        List(
+          "core.jar",
+          "core.docsJar",
+          "application.jar",
+          "application.docsJar"
+        )
+      )
+      'expandNested - {
+        check("{hello,world.{cow,moo}}",
+              List("hello", "world.cow", "world.moo"))
+        check("{a,b{c,d}}", List("a", "bc", "bd"))
+        check("{a,b,{c,d}}", List("a", "b", "c", "d"))
+        check("{a,b{c,d{e,f}}}", List("a", "bc", "bde", "bdf"))
+        check("{a{b,c},d}", List("ab", "ac", "d"))
+        check("{a,{b,c}d}", List("a", "bd", "cd"))
+        check("{a{b,c},d{e,f}}", List("ab", "ac", "de", "df"))
+        check("{a,b{c,d},e{f,g}}", List("a", "bc", "bd", "ef", "eg"))
+      }
+      'expandMixed - check(
+        "{a,b}.{c}.{}.e",
+        List("a.{c}.{}.e", "b.{c}.{}.e")
+      )
+      'malformed - {
+        val malformed = Seq("core.{compile", "core.{compile,test]")
+
+        malformed.foreach { m =>
+          val Left(error) = ParseArgs.expandBraces(m)
+          assert(error.contains("Parsing exception"))
+        }
+      }
+      'dontExpand - {
+        check("core.compile", List("core.compile"))
+        check("{}.compile", List("{}.compile"))
+        check("{core}.compile", List("{core}.compile"))
+      }
+      'keepUnknownSymbols - {
+        check("{a,b}.e<>", List("a.e<>", "b.e<>"))
+        check("a[99]&&", List("a[99]&&"))
+        check(
+          "{a,b}.<%%>.{c,d}",
+          List("a.<%%>.c", "a.<%%>.d", "b.<%%>.c", "b.<%%>.d")
+        )
+      }
+    }
+
+    'apply - {
+      def check(input: Seq[String],
+                expectedSelectors: List[List[Segment]],
+                expectedArgs: Seq[String]) = {
+        val Right((selectors, args)) = ParseArgs(input)
+
+        assert(
+          selectors == expectedSelectors,
+          args == expectedArgs
+        )
+      }
+
+      'rejectEmpty {
+        assert(ParseArgs(Seq.empty) == Left("Selector cannot be empty"))
+      }
+      'singleSelector - check(
+        input = Seq("core.compile"),
+        expectedSelectors = List(
+          List(Label("core"), Label("compile"))
+        ),
+        expectedArgs = Seq.empty
+      )
+      'singleSelectorWithArgs - check(
+        input = Seq("application.run", "hello", "world"),
+        expectedSelectors = List(
+          List(Label("application"), Label("run"))
+        ),
+        expectedArgs = Seq("hello", "world")
+      )
+      'singleSelectorWithCross - check(
+        input = Seq("bridges[2.12.4,jvm].compile"),
+        expectedSelectors = List(
+          List(Label("bridges"), Cross(Seq("2.12.4", "jvm")), Label("compile"))
+        ),
+        expectedArgs = Seq.empty
+      )
+      'multiSelectorsBraceExpansion - check(
+        input = Seq("--all", "{core,application}.compile"),
+        expectedSelectors = List(
+          List(Label("core"), Label("compile")),
+          List(Label("application"), Label("compile"))
+        ),
+        expectedArgs = Seq.empty
+      )
+      'multiSelectorsBraceExpansionWithArgs - check(
+        input = Seq("--all", "{core,application}.run", "--", "hello", "world"),
+        expectedSelectors = List(
+          List(Label("core"), Label("run")),
+          List(Label("application"), Label("run"))
+        ),
+        expectedArgs = Seq("hello", "world")
+      )
+      'multiSelectorsBraceExpansionWithCross - check(
+        input = Seq("--all", "bridges[2.12.4,jvm].{test,jar}"),
+        expectedSelectors = List(
+          List(Label("bridges"), Cross(Seq("2.12.4", "jvm")), Label("test")),
+          List(Label("bridges"), Cross(Seq("2.12.4", "jvm")), Label("jar"))
+        ),
+        expectedArgs = Seq.empty
+      )
+      'multiSelectorsBraceExpansionInsideCross - check(
+        input = Seq("--all", "bridges[{2.11.11,2.11.8}].jar"),
+        expectedSelectors = List(
+          List(Label("bridges"), Cross(Seq("2.11.11")), Label("jar")),
+          List(Label("bridges"), Cross(Seq("2.11.8")), Label("jar"))
+        ),
+        expectedArgs = Seq.empty
+      )
+      'multiSelectorsBraceExpansionWithoutAll - {
+        assert(
+          ParseArgs(Seq("{core,application}.compile")) == Left(
+            "Please use --all flag to run multiple tasks")
+        )
+      }
+      'multiSelectorsWithoutAllAsSingle - check( // this is how it works when we pass multiple tasks without --all flag
+        input = Seq("core.compile", "application.compile"),
+        expectedSelectors = List(
+          List(Label("core"), Label("compile"))
+        ),
+        expectedArgs = Seq("application.compile")
+      )
+    }
+  }
+
+}