From 7abb8cd2ce9df949723ab6a47c92e73c4e54d0fa Mon Sep 17 00:00:00 2001 From: Nikolay Tatarinov <5min4eq.unity@gmail.com> Date: Wed, 10 Jan 2018 00:19:01 +0300 Subject: ability to run multiple tasks via bash/zsh braces expansion (#104) fixes #31 --- core/src/main/scala/mill/main/ParseArgs.scala | 142 ++++++++++++++ core/src/main/scala/mill/main/RunScript.scala | 69 +++---- core/src/main/scala/mill/util/EitherOps.scala | 18 ++ core/src/test/scala/mill/main/MainTests.scala | 6 +- core/src/test/scala/mill/main/ParseArgsTest.scala | 229 ++++++++++++++++++++++ 5 files changed, 415 insertions(+), 49 deletions(-) create mode 100644 core/src/main/scala/mill/main/ParseArgs.scala create mode 100644 core/src/main/scala/mill/util/EitherOps.scala create mode 100644 core/src/test/scala/mill/main/ParseArgsTest.scala (limited to 'core') diff --git a/core/src/main/scala/mill/main/ParseArgs.scala b/core/src/main/scala/mill/main/ParseArgs.scala new file mode 100644 index 00000000..dc848418 --- /dev/null +++ b/core/src/main/scala/mill/main/ParseArgs.scala @@ -0,0 +1,142 @@ +package mill.main + +import mill.util.EitherOps +import fastparse.all._ +import mill.define.Segment + +object ParseArgs { + + def apply(scriptArgs: Seq[String]) + : Either[String, (List[List[Segment]], Seq[String])] = { + val (selectors, args, isMultiSelectors) = extractSelsAndArgs(scriptArgs) + for { + _ <- validateSelectors(selectors) + expandedSelectors <- EitherOps + .sequence(selectors.map(expandBraces)) + .map(_.flatten) + _ <- validateExpanded(expandedSelectors, isMultiSelectors) + selectors <- EitherOps.sequence(expandedSelectors.map(extractSegments)) + } yield (selectors.toList, args) + } + + def extractSelsAndArgs( + scriptArgs: Seq[String]): (Seq[String], Seq[String], Boolean) = { + val multiFlags = Seq("--all", "--seq") + val isMultiSelectors = scriptArgs.headOption.exists(multiFlags.contains) + + if (isMultiSelectors) { + val dd = scriptArgs.indexOf("--") + val selectors = (if (dd == -1) scriptArgs + else scriptArgs.take(dd)).filterNot(multiFlags.contains) + val args = if (dd == -1) Seq.empty else scriptArgs.drop(dd + 1) + + (selectors, args, isMultiSelectors) + } else { + (scriptArgs.take(1), scriptArgs.drop(1), isMultiSelectors) + } + } + + private def validateSelectors( + selectors: Seq[String]): Either[String, Unit] = { + if (selectors.isEmpty || selectors.exists(_.isEmpty)) + Left("Selector cannot be empty") + else Right(()) + } + + private def validateExpanded(expanded: Seq[String], + isMulti: Boolean): Either[String, Unit] = { + if (!isMulti && expanded.length > 1) + Left("Please use --all flag to run multiple tasks") + else Right(()) + } + + def expandBraces(selectorString: String): Either[String, List[String]] = { + parseBraceExpansion(selectorString) match { + case f: Parsed.Failure => Left(s"Parsing exception ${f.msg}") + case Parsed.Success(expanded, _) => Right(expanded.toList) + } + } + + private sealed trait Fragment + private object Fragment { + case class Keep(value: String) extends Fragment + case class Expand(values: List[List[Fragment]]) extends Fragment + + def unfold(fragments: List[Fragment]): Seq[String] = { + fragments match { + case head :: rest => + val prefixes = head match { + case Keep(v) => Seq(v) + case Expand(Nil) => Seq("{}") + case Expand(List(vs)) => unfold(vs).map("{" + _ + "}") + case Expand(vss) => vss.flatMap(unfold) + } + for { + prefix <- prefixes + suffix <- unfold(rest) + } yield prefix + suffix + + case Nil => Seq("") + } + } + } + + private object BraceExpansionParser { + val plainChars = + P(CharsWhile(c => c != ',' && c != '{' && c != '}')).!.map(Fragment.Keep) + + val toExpand: P[Fragment] = + P("{" ~ braceParser.rep(1).rep(sep = ",") ~ "}").map( + x => Fragment.Expand(x.toList.map(_.toList)) + ) + + val braceParser = P(toExpand | plainChars) + + val parser = P(braceParser.rep(1).rep(sep = ",") ~ End) + } + + private def parseBraceExpansion(input: String) = { + def unfold(vss: List[Seq[String]]): Seq[String] = { + vss match { + case Nil => Seq("") + case head :: rest => + for { + str <- head + r <- unfold(rest) + } yield + r match { + case "" => str + case _ => str + "," + r + } + } + } + + BraceExpansionParser.parser + .map { vss => + val stringss = vss.map(x => Fragment.unfold(x.toList)).toList + unfold(stringss) + } + .parse(input) + } + + def extractSegments(selectorString: String): Either[String, List[Segment]] = + parseSelector(selectorString) match { + case f: Parsed.Failure => Left(s"Parsing exception ${f.msg}") + case Parsed.Success(selector, _) => Right(selector) + } + + private def parseSelector(input: String) = { + val segment = + P(CharsWhileIn(('a' to 'z') ++ ('A' to 'Z') ++ ('0' to '9')).!).map( + Segment.Label + ) + val crossSegment = + P("[" ~ CharsWhile(c => c != ',' && c != ']').!.rep(1, sep = ",") ~ "]") + .map(Segment.Cross) + val query = P(segment ~ ("." ~ segment | crossSegment).rep ~ End).map { + case (h, rest) => h :: rest.toList + } + query.parse(input) + } + +} diff --git a/core/src/main/scala/mill/main/RunScript.scala b/core/src/main/scala/mill/main/RunScript.scala index e79349ad..b8793b40 100644 --- a/core/src/main/scala/mill/main/RunScript.scala +++ b/core/src/main/scala/mill/main/RunScript.scala @@ -1,6 +1,5 @@ package mill.main -import java.io.{ByteArrayOutputStream, PrintStream} import java.nio.file.NoSuchFileException import ammonite.interp.Interpreter @@ -10,9 +9,9 @@ import ammonite.util.{Name, Res, Util} import mill.{PathRef, define} import mill.define.Task import mill.define.Segment -import mill.discover.{Discovered, Mirror} -import mill.eval.{Evaluator, PathRef, Result} -import mill.util.{Logger, OSet, PrintLogger} +import mill.discover.Discovered +import mill.eval.{Evaluator, Result} +import mill.util.{EitherOps, Logger, OSet} import upickle.Js /** @@ -113,28 +112,32 @@ object RunScript{ _ <- Res(consistencyCheck(mapping)) } yield mapping } + def evaluateTarget[T](evaluator: Evaluator[_], scriptArgs: Seq[String]) = { - - val selectorString = scriptArgs.headOption.getOrElse("") - val rest = scriptArgs.drop(1) - for { - sel <- parseArgs(selectorString) - crossSelectors = sel.map{ - case Segment.Cross(x) => x.toList.map(_.toString) - case _ => Nil + parsed <- ParseArgs(scriptArgs) + (selectors, args) = parsed + targets <- { + val selected = selectors.map { sel => + val crossSelectors = sel.map { + case Segment.Cross(x) => x.toList.map(_.toString) + case _ => Nil + } + mill.main.Resolve.resolve( + sel, evaluator.mapping.mirror, evaluator.mapping.base, + args, crossSelectors, Nil + ) + } + EitherOps.sequence(selected) } - target <- mill.main.Resolve.resolve( - sel, evaluator.mapping.mirror, evaluator.mapping.base, - rest, crossSelectors, Nil - ) - (watched, res) = evaluate(evaluator, target) + (watched, res) = evaluate(evaluator, targets) } yield (watched, res) } + def evaluate(evaluator: Evaluator[_], - target: Task[Any]): (Seq[PathRef], Either[String, Seq[(Any, Option[upickle.Js.Value])]]) = { - val evaluated = evaluator.evaluate(OSet(target)) + targets: Seq[Task[Any]]): (Seq[PathRef], Either[String, Seq[(Any, Option[upickle.Js.Value])]]) = { + val evaluated = evaluator.evaluate(OSet.from(targets)) val watched = evaluated.results .iterator .collect { @@ -158,7 +161,7 @@ object RunScript{ evaluated.failing.keyCount match { case 0 => - val json = for(t <- Seq(target)) yield { + val json = for(t <- targets) yield { t match { case t: mill.define.NamedTask[_] => val jsonFile = Evaluator @@ -176,32 +179,6 @@ object RunScript{ } } - def parseSelector(input: String) = { - import fastparse.all._ - val segment = P( CharsWhileIn(('a' to 'z') ++ ('A' to 'Z') ++ ('0' to '9')).! ).map( - Segment.Label - ) - val crossSegment = P( "[" ~ CharsWhile(c => c != ',' && c != ']').!.rep(1, sep=",") ~ "]" ).map( - Segment.Cross - ) - val query = P( segment ~ ("." ~ segment | crossSegment).rep ~ End ).map{ - case (h, rest) => h :: rest.toList - } - query.parse(input) - } - - - - def parseArgs(selectorString: String): Either[String, List[Segment]] = { - import fastparse.all.Parsed - if (selectorString.isEmpty) Left("Selector cannot be empty") - else parseSelector(selectorString) match { - case f: Parsed.Failure => Left(s"Parsing exception ${f.msg}") - case Parsed.Success(selector, _) => Right(selector) - } - } - - def consistencyCheck[T](mapping: Discovered.Mapping[T]): Either[String, Unit] = { val consistencyErrors = Discovered.consistencyCheck(mapping) if (consistencyErrors.nonEmpty) { diff --git a/core/src/main/scala/mill/util/EitherOps.scala b/core/src/main/scala/mill/util/EitherOps.scala new file mode 100644 index 00000000..da2552c8 --- /dev/null +++ b/core/src/main/scala/mill/util/EitherOps.scala @@ -0,0 +1,18 @@ +package mill.util + +import scala.collection.generic.CanBuildFrom +import scala.collection.mutable +import scala.language.higherKinds + +object EitherOps { + + // implementation similar to scala.concurrent.Future#sequence + def sequence[A, B, M[X] <: TraversableOnce[X]](in: M[Either[A, B]])( + implicit cbf: CanBuildFrom[M[Either[A, B]], B, M[B]]): Either[A, M[B]] = { + in.foldLeft[Either[A, mutable.Builder[B, M[B]]]](Right(cbf(in))) { + case (acc, el) => + for (a <- acc; e <- el) yield a += e + } + .map(_.result()) + } +} diff --git a/core/src/test/scala/mill/main/MainTests.scala b/core/src/test/scala/mill/main/MainTests.scala index 013729ff..b4568267 100644 --- a/core/src/test/scala/mill/main/MainTests.scala +++ b/core/src/test/scala/mill/main/MainTests.scala @@ -13,9 +13,9 @@ object MainTests extends TestSuite{ expected: Either[String, Task[_]]) = { val resolved = for{ - args <- mill.main.RunScript.parseArgs(selectorString) - val crossSelectors = args.map{case Segment.Cross(x) => x.toList.map(_.toString) case _ => Nil} - task <- mill.main.Resolve.resolve(args, mapping.mirror, mapping.base, Nil, crossSelectors, Nil) + selectors <- mill.main.ParseArgs(Seq(selectorString)).map(_._1.head) + val crossSelectors = selectors.map{case Segment.Cross(x) => x.toList.map(_.toString) case _ => Nil} + task <- mill.main.Resolve.resolve(selectors, mapping.mirror, mapping.base, Nil, crossSelectors, Nil) } yield task assert(resolved == expected) } diff --git a/core/src/test/scala/mill/main/ParseArgsTest.scala b/core/src/test/scala/mill/main/ParseArgsTest.scala new file mode 100644 index 00000000..2ef07d36 --- /dev/null +++ b/core/src/test/scala/mill/main/ParseArgsTest.scala @@ -0,0 +1,229 @@ +package mill.main + +import mill.define.Segment +import mill.define.Segment.{Cross, Label} +import utest._ + +object ParseArgsTest extends TestSuite { + + val tests = Tests { + 'extractSelsAndArgs - { + def check(input: Seq[String], + expectedSelectors: Seq[String], + expectedArgs: Seq[String], + expectedIsMulti: Boolean) = { + val (selectors, args, isMulti) = ParseArgs.extractSelsAndArgs(input) + + assert( + selectors == expectedSelectors, + args == expectedArgs, + isMulti == expectedIsMulti + ) + } + + 'empty - check(input = Seq.empty, + expectedSelectors = Seq.empty, + expectedArgs = Seq.empty, + expectedIsMulti = false) + 'singleSelector - check( + input = Seq("core.compile"), + expectedSelectors = Seq("core.compile"), + expectedArgs = Seq.empty, + expectedIsMulti = false + ) + 'singleSelectorWithArgs - check( + input = Seq("application.run", "hello", "world"), + expectedSelectors = Seq("application.run"), + expectedArgs = Seq("hello", "world"), + expectedIsMulti = false + ) + 'singleSelectorWithAllInArgs - check( + input = Seq("application.run", "hello", "world", "--all"), + expectedSelectors = Seq("application.run"), + expectedArgs = Seq("hello", "world", "--all"), + expectedIsMulti = false + ) + 'multiSelectors - check( + input = Seq("--all", "core.jar", "core.docsJar", "core.sourcesJar"), + expectedSelectors = Seq("core.jar", "core.docsJar", "core.sourcesJar"), + expectedArgs = Seq.empty, + expectedIsMulti = true + ) + 'multiSelectorsSeq - check( + input = Seq("--seq", "core.jar", "core.docsJar", "core.sourcesJar"), + expectedSelectors = Seq("core.jar", "core.docsJar", "core.sourcesJar"), + expectedArgs = Seq.empty, + expectedIsMulti = true + ) + 'multiSelectorsWithArgs - check( + input = Seq("--all", + "core.compile", + "application.runMain", + "--", + "Main", + "hello", + "world"), + expectedSelectors = Seq("core.compile", "application.runMain"), + expectedArgs = Seq("Main", "hello", "world"), + expectedIsMulti = true + ) + 'multiSelectorsWithArgsWithAllInArgs - check( + input = Seq("--all", + "core.compile", + "application.runMain", + "--", + "Main", + "--all", + "world"), + expectedSelectors = Seq("core.compile", "application.runMain"), + expectedArgs = Seq("Main", "--all", "world"), + expectedIsMulti = true + ) + } + 'expandBraces - { + def check(input: String, expectedExpansion: List[String]) = { + val Right(expanded) = ParseArgs.expandBraces(input) + + assert(expanded == expectedExpansion) + } + + 'expandLeft - check( + "{application,core}.compile", + List("application.compile", "core.compile") + ) + 'expandRight - check( + "application.{jar,docsJar,sourcesJar}", + List("application.jar", "application.docsJar", "application.sourcesJar") + ) + 'expandBoth - check( + "{core,application}.{jar,docsJar}", + List( + "core.jar", + "core.docsJar", + "application.jar", + "application.docsJar" + ) + ) + 'expandNested - { + check("{hello,world.{cow,moo}}", + List("hello", "world.cow", "world.moo")) + check("{a,b{c,d}}", List("a", "bc", "bd")) + check("{a,b,{c,d}}", List("a", "b", "c", "d")) + check("{a,b{c,d{e,f}}}", List("a", "bc", "bde", "bdf")) + check("{a{b,c},d}", List("ab", "ac", "d")) + check("{a,{b,c}d}", List("a", "bd", "cd")) + check("{a{b,c},d{e,f}}", List("ab", "ac", "de", "df")) + check("{a,b{c,d},e{f,g}}", List("a", "bc", "bd", "ef", "eg")) + } + 'expandMixed - check( + "{a,b}.{c}.{}.e", + List("a.{c}.{}.e", "b.{c}.{}.e") + ) + 'malformed - { + val malformed = Seq("core.{compile", "core.{compile,test]") + + malformed.foreach { m => + val Left(error) = ParseArgs.expandBraces(m) + assert(error.contains("Parsing exception")) + } + } + 'dontExpand - { + check("core.compile", List("core.compile")) + check("{}.compile", List("{}.compile")) + check("{core}.compile", List("{core}.compile")) + } + 'keepUnknownSymbols - { + check("{a,b}.e<>", List("a.e<>", "b.e<>")) + check("a[99]&&", List("a[99]&&")) + check( + "{a,b}.<%%>.{c,d}", + List("a.<%%>.c", "a.<%%>.d", "b.<%%>.c", "b.<%%>.d") + ) + } + } + + 'apply - { + def check(input: Seq[String], + expectedSelectors: List[List[Segment]], + expectedArgs: Seq[String]) = { + val Right((selectors, args)) = ParseArgs(input) + + assert( + selectors == expectedSelectors, + args == expectedArgs + ) + } + + 'rejectEmpty { + assert(ParseArgs(Seq.empty) == Left("Selector cannot be empty")) + } + 'singleSelector - check( + input = Seq("core.compile"), + expectedSelectors = List( + List(Label("core"), Label("compile")) + ), + expectedArgs = Seq.empty + ) + 'singleSelectorWithArgs - check( + input = Seq("application.run", "hello", "world"), + expectedSelectors = List( + List(Label("application"), Label("run")) + ), + expectedArgs = Seq("hello", "world") + ) + 'singleSelectorWithCross - check( + input = Seq("bridges[2.12.4,jvm].compile"), + expectedSelectors = List( + List(Label("bridges"), Cross(Seq("2.12.4", "jvm")), Label("compile")) + ), + expectedArgs = Seq.empty + ) + 'multiSelectorsBraceExpansion - check( + input = Seq("--all", "{core,application}.compile"), + expectedSelectors = List( + List(Label("core"), Label("compile")), + List(Label("application"), Label("compile")) + ), + expectedArgs = Seq.empty + ) + 'multiSelectorsBraceExpansionWithArgs - check( + input = Seq("--all", "{core,application}.run", "--", "hello", "world"), + expectedSelectors = List( + List(Label("core"), Label("run")), + List(Label("application"), Label("run")) + ), + expectedArgs = Seq("hello", "world") + ) + 'multiSelectorsBraceExpansionWithCross - check( + input = Seq("--all", "bridges[2.12.4,jvm].{test,jar}"), + expectedSelectors = List( + List(Label("bridges"), Cross(Seq("2.12.4", "jvm")), Label("test")), + List(Label("bridges"), Cross(Seq("2.12.4", "jvm")), Label("jar")) + ), + expectedArgs = Seq.empty + ) + 'multiSelectorsBraceExpansionInsideCross - check( + input = Seq("--all", "bridges[{2.11.11,2.11.8}].jar"), + expectedSelectors = List( + List(Label("bridges"), Cross(Seq("2.11.11")), Label("jar")), + List(Label("bridges"), Cross(Seq("2.11.8")), Label("jar")) + ), + expectedArgs = Seq.empty + ) + 'multiSelectorsBraceExpansionWithoutAll - { + assert( + ParseArgs(Seq("{core,application}.compile")) == Left( + "Please use --all flag to run multiple tasks") + ) + } + 'multiSelectorsWithoutAllAsSingle - check( // this is how it works when we pass multiple tasks without --all flag + input = Seq("core.compile", "application.compile"), + expectedSelectors = List( + List(Label("core"), Label("compile")) + ), + expectedArgs = Seq("application.compile") + ) + } + } + +} -- cgit v1.2.3