From 68f158b63c33aa7312ba4b715f6b18527453b761 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 21 Jan 2018 16:11:14 -0800 Subject: Implement basic wildcard task running via e.g. `mill _.compile` --- core/src/mill/define/Module.scala | 5 +- core/src/mill/define/Task.scala | 3 +- core/src/mill/main/ParseArgs.scala | 12 +- core/src/mill/main/Resolve.scala | 64 +++++-- core/src/mill/main/RunScript.scala | 11 +- core/test/src/mill/main/MainTests.scala | 210 +++++++++++++++++---- readme.md | 20 +- scalalib/test/src/mill/scalalib/GenIdeaTests.scala | 1 - 8 files changed, 263 insertions(+), 63 deletions(-) diff --git a/core/src/mill/define/Module.scala b/core/src/mill/define/Module.scala index e42ce798..c68c2bdd 100644 --- a/core/src/mill/define/Module.scala +++ b/core/src/mill/define/Module.scala @@ -37,8 +37,9 @@ object Module{ def rec(m: Module): Seq[T] = f(m) ++ m.millModuleDirectChildren.flatMap(rec) rec(outer) } - lazy val segmentsToModules = traverse{m => Seq(m.millModuleSegments -> m)} - .toMap + + lazy val modules = traverse(Seq(_)) + lazy val segmentsToModules = modules.map(m => (m.millModuleSegments, m)).toMap lazy val targets = traverse{_.millInternal.reflect[Target[_]]}.toSet diff --git a/core/src/mill/define/Task.scala b/core/src/mill/define/Task.scala index 90908e4e..248f145c 100644 --- a/core/src/mill/define/Task.scala +++ b/core/src/mill/define/Task.scala @@ -43,6 +43,7 @@ abstract class Task[+T] extends Task.Ops[T] with Applyable[Task, T]{ trait NamedTask[+T] extends Task[T]{ def ctx: mill.define.Ctx def label = ctx.segment match{case Segment.Label(v) => v} + override def toString = ctx.segments.render } trait Target[+T] extends NamedTask[T]{ override def asTarget = Some(this) @@ -232,7 +233,7 @@ class TargetImpl[+T](t: Task[T], val ctx = ctx0.copy(segments = ctx0.segments ++ Seq(ctx0.segment)) val inputs = Seq(t) def evaluate(args: mill.util.Ctx) = args[T](0) - override def toString = ctx.enclosing + "@" + Integer.toHexString(System.identityHashCode(this)) + } class Command[+T](t: Task[T], ctx0: mill.define.Ctx, diff --git a/core/src/mill/main/ParseArgs.scala b/core/src/mill/main/ParseArgs.scala index dc848418..7170cc60 100644 --- a/core/src/mill/main/ParseArgs.scala +++ b/core/src/mill/main/ParseArgs.scala @@ -126,13 +126,11 @@ object ParseArgs { } private def parseSelector(input: String) = { - val segment = - P(CharsWhileIn(('a' to 'z') ++ ('A' to 'Z') ++ ('0' to '9')).!).map( - Segment.Label - ) - val crossSegment = - P("[" ~ CharsWhile(c => c != ',' && c != ']').!.rep(1, sep = ",") ~ "]") - .map(Segment.Cross) + val identChars = ('a' to 'z') ++ ('A' to 'Z') ++ ('0' to '9') ++ "_" + val ident = P( CharsWhileIn(identChars) ).! + val ident2 = P( CharsWhileIn(identChars ++ ".") ).! + val segment = P( ident ).map( Segment.Label) + val crossSegment = P("[" ~ ident2.rep(1, sep = ",") ~ "]").map(Segment.Cross) val query = P(segment ~ ("." ~ segment | crossSegment).rep ~ End).map { case (h, rest) => h :: rest.toList } diff --git a/core/src/mill/main/Resolve.scala b/core/src/mill/main/Resolve.scala index 3d9c0409..d836a1a1 100644 --- a/core/src/mill/main/Resolve.scala +++ b/core/src/mill/main/Resolve.scala @@ -11,7 +11,7 @@ object Resolve { discover: Discover, rest: Seq[String], remainingCrossSelectors: List[List[String]], - revSelectorsSoFar: List[Segment]): Either[String, Task[Any]] = { + revSelectorsSoFar: List[Segment]): Either[String, Seq[Task[Any]]] = { remainingSelector match{ case Segment.Cross(_) :: Nil => Left("Selector cannot start with a [cross] segment") @@ -23,7 +23,7 @@ object Resolve { .find(_.label == last) .map(Right(_)) - def invokeCommand[V](target: mill.Module, name: String) = { + def invokeCommand(target: mill.Module, name: String) = { for{ (cls, entryPoints) <- discover.value.filterKeys(_.isAssignableFrom(target.getClass)) @@ -52,7 +52,7 @@ object Resolve { ) // Contents of `either` *must* be a `Task`, because we only select // methods returning `Task` in the discovery process - case Some(either) => either.right.map{ case x: Task[Any] => x } + case Some(either) => either.right.map{ case x: Task[Any] => Seq(x) } } @@ -60,20 +60,62 @@ object Resolve { val newRevSelectorsSoFar = head :: revSelectorsSoFar head match{ case Segment.Label(singleLabel) => - obj.millInternal.reflectNestedObjects[mill.Module].find{ - _.millOuterCtx.segment == Segment.Label(singleLabel) - } match{ - case Some(child: mill.Module) => resolve(tail, child, discover, rest, remainingCrossSelectors, newRevSelectorsSoFar) - case None => Left("Cannot resolve module " + Segments(newRevSelectorsSoFar.reverse:_*).render) + if (singleLabel == "__"){ + val matching = + obj.millInternal + .modules + .map(resolve(tail, _, discover, rest, remainingCrossSelectors, newRevSelectorsSoFar)) + .collect{case Right(vs) => vs}.flatten + + if (matching.nonEmpty) Right(matching.toSeq) + else Left("Cannot resolve module " + Segments(newRevSelectorsSoFar.reverse:_*).render) + }else if (singleLabel == "_") { + val matching = + obj.millInternal + .reflectNestedObjects[mill.Module] + .map(resolve(tail, _, discover, rest, remainingCrossSelectors, newRevSelectorsSoFar)) + .collect{case Right(vs) => vs}.flatten + + if (matching.nonEmpty)Right(matching) + else Left("Cannot resolve module " + Segments(newRevSelectorsSoFar.reverse:_*).render) + }else{ + + obj.millInternal.reflectNestedObjects[mill.Module].find{ + _.millOuterCtx.segment == Segment.Label(singleLabel) + } match{ + case Some(child: mill.Module) => resolve(tail, child, discover, rest, remainingCrossSelectors, newRevSelectorsSoFar) + case None => Left("Cannot resolve module " + Segments(newRevSelectorsSoFar.reverse:_*).render) + } } case Segment.Cross(cross) => obj match{ case c: Cross[_] => - c.itemMap.get(cross.toList) match{ - case Some(m: mill.Module) => resolve(tail, m, discover, rest, remainingCrossSelectors, newRevSelectorsSoFar) - case None => Left("Cannot resolve cross " + Segments(newRevSelectorsSoFar.reverse:_*).render) + if(cross == Seq("__")){ + val matching = + for ((k, v) <- c.items) + yield resolve(tail, v.asInstanceOf[mill.Module], discover, rest, remainingCrossSelectors, newRevSelectorsSoFar) + + val results = matching.collect{case Right(res) => res}.flatten + + if (results.isEmpty) Left("Cannot resolve cross " + Segments(newRevSelectorsSoFar.reverse:_*).render) + else Right(results) + } else if (cross.contains("_")){ + val matching = for { + (k, v) <- c.items + if k.length == cross.length + if k.zip(cross).forall { case (l, r) => l == r || r == "_" } + } yield resolve(tail, v.asInstanceOf[mill.Module], discover, rest, remainingCrossSelectors, newRevSelectorsSoFar) + + val results = matching.collect{case Right(res) => res}.flatten + if (results.isEmpty) Left("Cannot resolve cross " + Segments(newRevSelectorsSoFar.reverse:_*).render) + else Right(results) + }else{ + c.itemMap.get(cross.toList) match{ + case Some(m: mill.Module) => resolve(tail, m, discover, rest, remainingCrossSelectors, newRevSelectorsSoFar) + case None => Left("Cannot resolve cross " + Segments(newRevSelectorsSoFar.reverse:_*).render) + } } case _ => Left("Cannot resolve cross " + Segments(newRevSelectorsSoFar.reverse:_*).render) } diff --git a/core/src/mill/main/RunScript.scala b/core/src/mill/main/RunScript.scala index aa254048..a69f87be 100644 --- a/core/src/mill/main/RunScript.scala +++ b/core/src/mill/main/RunScript.scala @@ -151,13 +151,16 @@ object RunScript{ } EitherOps.sequence(selected) } - (watched, res) = evaluate(evaluator, targets) + (watched, res) = evaluate( + evaluator, + Agg.from(targets.flatten.distinct) + ) } yield (watched, res) } def evaluate(evaluator: Evaluator[_], - targets: Seq[Task[Any]]): (Seq[PathRef], Either[String, Seq[(Any, Option[upickle.Js.Value])]]) = { - val evaluated = evaluator.evaluate(Agg.from(targets)) + targets: Agg[Task[Any]]): (Seq[PathRef], Either[String, Seq[(Any, Option[upickle.Js.Value])]]) = { + val evaluated = evaluator.evaluate(targets) val watched = evaluated.results .iterator .collect { @@ -194,7 +197,7 @@ object RunScript{ } } - watched -> Right(evaluated.values.zip(json)) + watched -> Right(evaluated.values.zip(json.toSeq)) case n => watched -> Left(s"$n targets failed\n$errorStr") } } diff --git a/core/test/src/mill/main/MainTests.scala b/core/test/src/mill/main/MainTests.scala index c2499835..96fd4448 100644 --- a/core/test/src/mill/main/MainTests.scala +++ b/core/test/src/mill/main/MainTests.scala @@ -6,11 +6,13 @@ import mill.util.TestGraphs._ import mill.util.TestUtil.test import utest._ object MainTests extends TestSuite{ - def check[T](module: mill.Module, - discover: Discover, - selectorString: String, - expected: Either[String, Task[_]]) = { + def check[T <: mill.Module](module: T, + discover: Discover)( + selectorString: String, + expected0: Either[String, Seq[T => Task[_]]]) = { + + val expected = expected0.map(_.map(_(module))) val resolved = for{ selectors <- mill.main.ParseArgs(Seq(selectorString)).map(_._1.head) val crossSelectors = selectors.map{case Segment.Cross(x) => x.toList.map(_.toString) case _ => Nil} @@ -22,60 +24,198 @@ object MainTests extends TestSuite{ val graphs = new mill.util.TestGraphs() import graphs._ 'single - { - 'pos - check(singleton, Discover[singleton.type], "single", Right(singleton.single)) - 'neg1 - check(singleton, Discover[singleton.type], "doesntExist", Left("Cannot resolve task doesntExist")) - 'neg2 - check(singleton, Discover[singleton.type], "single.doesntExist", Left("Cannot resolve module single")) - 'neg3 - check(singleton, Discover[singleton.type], "", Left("Selector cannot be empty")) + val check = MainTests.check(singleton, Discover[singleton.type]) _ + 'pos - check("single", Right(Seq(_.single))) + 'neg1 - check("doesntExist", Left("Cannot resolve task doesntExist")) + 'neg2 - check("single.doesntExist", Left("Cannot resolve module single")) + 'neg3 - check("", Left("Selector cannot be empty")) } 'nested - { + val check = MainTests.check(nestedModule, Discover[nestedModule.type]) _ + 'pos1 - check("single", Right(Seq(_.single))) + 'pos2 - check("nested.single", Right(Seq(_.nested.single))) + 'pos3 - check("classInstance.single", Right(Seq(_.classInstance.single))) + 'neg1 - check("doesntExist", Left("Cannot resolve task doesntExist")) + 'neg2 - check("single.doesntExist", Left("Cannot resolve module single")) + 'neg3 - check("nested.doesntExist", Left("Cannot resolve task nested.doesntExist")) + 'neg4 - check("classInstance.doesntExist", Left("Cannot resolve task classInstance.doesntExist")) + 'wildcard - check( + "_.single", + Right(Seq( + _.classInstance.single, + _.nested.single + )) + ) + 'wildcardNeg - check( + "_._.single", + Left("Cannot resolve module _") + ) + 'wildcardNeg2 - check( + "_._.__", + Left("Cannot resolve module _") + ) + 'wildcard2 - check( + "__.single", + Right(Seq( + _.single, + _.classInstance.single, + _.nested.single + )) + ) + + 'wildcard3 - check( + "_.__.single", + Right(Seq( + _.classInstance.single, + _.nested.single + )) + ) - 'pos1 - check(nestedModule, Discover[nestedModule.type], "single", Right(nestedModule.single)) - 'pos2 - check(nestedModule, Discover[nestedModule.type], "nested.single", Right(nestedModule.nested.single)) - 'pos3 - check(nestedModule, Discover[nestedModule.type], "classInstance.single", Right(nestedModule.classInstance.single)) - 'neg1 - check(nestedModule, Discover[nestedModule.type], "doesntExist", Left("Cannot resolve task doesntExist")) - 'neg2 - check(nestedModule, Discover[nestedModule.type], "single.doesntExist", Left("Cannot resolve module single")) - 'neg3 - check(nestedModule, Discover[nestedModule.type], "nested.doesntExist", Left("Cannot resolve task nested.doesntExist")) - 'neg4 - check(nestedModule, Discover[nestedModule.type], "classInstance.doesntExist", Left("Cannot resolve task classInstance.doesntExist")) } 'cross - { 'single - { - - 'pos1 - check(singleCross, Discover[singleCross.type], "cross[210].suffix", Right(singleCross.cross("210").suffix)) - 'pos2 - check(singleCross, Discover[singleCross.type], "cross[211].suffix", Right(singleCross.cross("211").suffix)) - 'neg1 - check(singleCross, Discover[singleCross.type], "cross[210].doesntExist", Left("Cannot resolve task cross[210].doesntExist")) - 'neg2 - check(singleCross, Discover[singleCross.type], "cross[doesntExist].doesntExist", Left("Cannot resolve cross cross[doesntExist]")) - 'neg2 - check(singleCross, Discover[singleCross.type], "cross[doesntExist].suffix", Left("Cannot resolve cross cross[doesntExist]")) + val check = MainTests.check(singleCross, Discover[singleCross.type]) _ + 'pos1 - check("cross[210].suffix", Right(Seq(_.cross("210").suffix))) + 'pos2 - check("cross[211].suffix", Right(Seq(_.cross("211").suffix))) + 'neg1 - check("cross[210].doesntExist", Left("Cannot resolve task cross[210].doesntExist")) + 'neg2 - check("cross[doesntExist].doesntExist", Left("Cannot resolve cross cross[doesntExist]")) + 'neg2 - check("cross[doesntExist].suffix", Left("Cannot resolve cross cross[doesntExist]")) + 'wildcard - check( + "cross[_].suffix", + Right(Seq( + _.cross("210").suffix, + _.cross("211").suffix, + _.cross("212").suffix + )) + ) + 'wildcard2 - check( + "cross[__].suffix", + Right(Seq( + _.cross("210").suffix, + _.cross("211").suffix, + _.cross("212").suffix + )) + ) } 'double - { - + val check = MainTests.check(doubleCross, Discover[doubleCross.type]) _ 'pos1 - check( - doubleCross, - Discover[doubleCross.type], "cross[210,jvm].suffix", - Right(doubleCross.cross("210", "jvm").suffix) + Right(Seq(_.cross("210", "jvm").suffix)) ) 'pos2 - check( - doubleCross, - Discover[doubleCross.type], "cross[211,jvm].suffix", - Right(doubleCross.cross("211", "jvm").suffix) + Right(Seq(_.cross("211", "jvm").suffix)) ) + 'wildcard - { + 'labelNeg - check( + "_.suffix", + Left("Cannot resolve module _") + ) + 'labelPos - check( + "__.suffix", + Right(Seq( + _.cross("210", "jvm").suffix, + _.cross("210", "js").suffix, + + _.cross("211", "jvm").suffix, + _.cross("211", "js").suffix, + + _.cross("212", "jvm").suffix, + _.cross("212", "js").suffix, + _.cross("212", "native").suffix + )) + ) + 'first - check( + "cross[_,jvm].suffix", + Right(Seq( + _.cross("210", "jvm").suffix, + _.cross("211", "jvm").suffix, + _.cross("212", "jvm").suffix + )) + ) + 'second - check( + "cross[210,_].suffix", + Right(Seq( + _.cross("210", "jvm").suffix, + _.cross("210", "js").suffix + )) + ) + 'both - check( + "cross[_,_].suffix", + Right(Seq( + _.cross("210", "jvm").suffix, + _.cross("210", "js").suffix, + + _.cross("211", "jvm").suffix, + _.cross("211", "js").suffix, + + _.cross("212", "jvm").suffix, + _.cross("212", "js").suffix, + _.cross("212", "native").suffix + )) + ) + 'both2 - check( + "cross[__].suffix", + Right(Seq( + _.cross("210", "jvm").suffix, + _.cross("210", "js").suffix, + + _.cross("211", "jvm").suffix, + _.cross("211", "js").suffix, + + _.cross("212", "jvm").suffix, + _.cross("212", "js").suffix, + _.cross("212", "native").suffix + )) + ) + } } 'nested - { + val check = MainTests.check(nestedCrosses, Discover[nestedCrosses.type]) _ 'pos1 - check( - nestedCrosses, - Discover[nestedCrosses.type], "cross[210].cross2[js].suffix", - Right(nestedCrosses.cross("210").cross2("js").suffix) + Right(Seq(_.cross("210").cross2("js").suffix)) ) 'pos2 - check( - nestedCrosses, - Discover[nestedCrosses.type], "cross[211].cross2[jvm].suffix", - Right(nestedCrosses.cross("211").cross2("jvm").suffix) + Right(Seq(_.cross("211").cross2("jvm").suffix)) ) + 'wildcard - { + 'first - check( + "cross[_].cross2[jvm].suffix", + Right(Seq( + _.cross("210").cross2("jvm").suffix, + _.cross("211").cross2("jvm").suffix, + _.cross("212").cross2("jvm").suffix + )) + ) + 'second - check( + "cross[210].cross2[_].suffix", + Right(Seq( + _.cross("210").cross2("jvm").suffix, + _.cross("210").cross2("js").suffix, + _.cross("210").cross2("native").suffix + )) + ) + 'both - check( + "cross[_].cross2[_].suffix", + Right(Seq( + _.cross("210").cross2("jvm").suffix, + _.cross("210").cross2("js").suffix, + _.cross("210").cross2("native").suffix, + + _.cross("211").cross2("jvm").suffix, + _.cross("211").cross2("js").suffix, + _.cross("211").cross2("native").suffix, + + _.cross("212").cross2("jvm").suffix, + _.cross("212").cross2("js").suffix, + _.cross("212").cross2("native").suffix + )) + ) + } } } - } } diff --git a/readme.md b/readme.md index f8ee9a21..b1ef0ff8 100644 --- a/readme.md +++ b/readme.md @@ -126,14 +126,30 @@ mill --show --all {core,scalalib}.{scalaVersion,scalaBinaryVersion} will run `scalaVersion` and `scalaBinaryVersion` targets in both `core` and `scalalib` modules. * Run targets in different cross build modules + ```bash mill --all bridges[{2.11.11,2.12.4}].publish -- --credentials foo --gpgPassphrase bar ``` will run `publish` command in both `brides[2.11.11]` and `bridges[2.12.4]` modules -**Note**: When you run multiple targets with `--all` flag, they are not guaranteed to run in that exact order. -Mill will build task evaluation graph and run targets in correct order. +You can also use the `_` wildcard and `__` recursive-wildcard to run groups of +tasks: + +```bash +# Run the `test` command of all top-level modules +mill --all _.test + +# Run the `test` command of all modules, top-level or nested +mill --all __.test + +# Run `compile` in every cross-module of `bridges` +mill --all bridges[_].compile +``` + +**Note**: When you run multiple targets with `--all` flag, they are not +guaranteed to run in that exact order. Mill will build task evaluation graph and +run targets in correct order. ### REPL diff --git a/scalalib/test/src/mill/scalalib/GenIdeaTests.scala b/scalalib/test/src/mill/scalalib/GenIdeaTests.scala index 36945f65..3bafcfe3 100644 --- a/scalalib/test/src/mill/scalalib/GenIdeaTests.scala +++ b/scalalib/test/src/mill/scalalib/GenIdeaTests.scala @@ -61,7 +61,6 @@ object GenIdeaTests extends TestSuite { "gen-idea/idea/misc.xml" -> basePath / "generated" / ".idea" / "misc.xml", ).foreach { case (resource, generated) => - println("checking "+resource) val resourceString = scala.io.Source.fromResource(resource).getLines().mkString("\n") val generatedString = normaliseLibraryPaths(read! generated) -- cgit v1.2.3