SI-5920 enables default and named args in macros

When producing an initial spec for macros two years ago, we sort of glossed
over named/default arguments in macro applications, leaving them for future work.

Once the aforementioned future has come, I’ve made several attempts at
making things operational (e.g. last summer), but it’s always been unclear
how to marry the quite complex desugaring that tryNamesDefaults performs
with the expectations of macro programmers to see unsugared trees
in macro impl parameters.

Here’s the list of problems that arise when trying to encode named/default
arguments of macro applications:
1) When inside macro impls we don’t really care about synthetic vals
that are typically introduced to preserve evaluation order in non-positional
method applications. When we inline those synthetics, we lose information
about evaluation order, which is something that we wouldn’t like to lose
in the general case.
2) More importantly, it’s also not very exciting to see invocations of
default getters that stand for unspecified default arguments. Ideally,
we would like to provide macro programmers with right-hand sides of those
default getters, but that is: a) impossible in the current implementation
of default parameters, b) would anyway bring scoping problems that we’re
not ready to deal with just yet.

Being constantly unhappy with potential solutions to the aforementioned
problems, I’ve been unable to nail this down until the last weekend,
when I realized that: 1) even though we can’t express potential twists in
evaluation order within linearly ordered macro impl params, we can use
c.macroApplication to store all the named arguments we want, 2) even though
we can’t get exactly what we want for default arguments, we can represent
them with EmptyTree’s, which is not ideal, but pretty workable. That’s
what has been put into life in this commit.

As a pleasant side-effect, now the macro engine doesn’t have to reinvent
the wheel wrt reporting errors about insufficient arg or arglist count.
Since this logic is intertwined with the tryNamesDefaults desugaring,
we previously couldn’t make use of it and had to roll our own logic
that checked that the number of arguments and parameters of macro applications
correspond to each other. Now it’s all deduplicated and consistent.

5 files changed, 88 insertions, 66 deletions

diff --git a/src/compiler/scala/tools/nsc/typechecker/ContextErrors.scala b/src/compiler/scala/tools/nsc/typechecker/ContextErrors.scala
index 2043eb5d5d..40b97394f2 100644
--- a/src/compiler/scala/tools/nsc/typechecker/ContextErrors.scala
+++ b/src/compiler/scala/tools/nsc/typechecker/ContextErrors.scala
@@ -501,10 +501,6 @@ trait ContextErrors {
       }
 
       // doTypeApply
-      //tryNamesDefaults
-      def NamedAndDefaultArgumentsNotSupportedForMacros(tree: Tree, fun: Tree) =
-        NormalTypeError(tree, "macro applications do not support named and/or default arguments")
-
       def TooManyArgsNamesDefaultsError(tree: Tree, fun: Tree) =
         NormalTypeError(tree, "too many arguments for "+treeSymTypeMsg(fun))
 
@@ -603,12 +599,11 @@ trait ContextErrors {
 
       //adapt
       def MissingArgsForMethodTpeError(tree: Tree, meth: Symbol) = {
+        val errorExplanation = "missing arguments for " + meth.fullLocationString
+        val suggestPartialApplication = ";\nfollow this method with `_' if you want to treat it as a partially applied function"
         val message =
-          if (meth.isMacro) MacroTooFewArgumentListsMessage
-          else "missing arguments for " + meth.fullLocationString + (
-            if (meth.isConstructor) ""
-            else ";\nfollow this method with `_' if you want to treat it as a partially applied function"
-          )
+          if (meth.isMacro || meth.isConstructor) errorExplanation
+          else errorExplanation + suggestPartialApplication
         issueNormalTypeError(tree, message)
         setError(tree)
       }
@@ -748,15 +743,12 @@ trait ContextErrors {
         throw MacroExpansionException
       }
 
-      private def MacroTooFewArgumentListsMessage = "too few argument lists for macro invocation"
-      def MacroTooFewArgumentListsError(expandee: Tree) = macroExpansionError2(expandee, MacroTooFewArgumentListsMessage)
-
-      private def MacroTooManyArgumentListsMessage = "too many argument lists for macro invocation"
-      def MacroTooManyArgumentListsError(expandee: Tree) = macroExpansionError2(expandee, MacroTooManyArgumentListsMessage)
-
-      def MacroTooFewArgumentsError(expandee: Tree) = macroExpansionError2(expandee, "too few arguments for macro invocation")
-
-      def MacroTooManyArgumentsError(expandee: Tree) = macroExpansionError2(expandee, "too many arguments for macro invocation")
+      def MacroFastTrackFailed(expandee: Tree) = {
+        // here we speculate that the reason why FastTrackEntry.validate failed is the lack arguments for a given method
+        // that's not ideal, but on the other hand this allows us to keep FastTrack simple without hooking errorgen into it
+        MissingArgsForMethodTpeError(expandee, expandee.symbol)
+        throw MacroExpansionException
+      }
 
       def MacroGeneratedAbort(expandee: Tree, ex: AbortMacroException) = {
         // errors have been reported by the macro itself, so we do nothing here
diff --git a/src/compiler/scala/tools/nsc/typechecker/Macros.scala b/src/compiler/scala/tools/nsc/typechecker/Macros.scala
index cf82d6baac..29b3ec7f3e 100644
--- a/src/compiler/scala/tools/nsc/typechecker/Macros.scala
+++ b/src/compiler/scala/tools/nsc/typechecker/Macros.scala
@@ -353,7 +353,7 @@ trait Macros extends FastTrack with MacroRuntimes with Traces with Helpers {
     new {
       val universe: self.global.type = self.global
       val callsiteTyper: universe.analyzer.Typer = typer.asInstanceOf[global.analyzer.Typer]
-      val expandee = universe.analyzer.macroExpanderAttachment(expandeeTree).original orElse duplicateAndKeepPositions(expandeeTree)
+      val expandee = universe.analyzer.macroExpanderAttachment(expandeeTree).desugared orElse duplicateAndKeepPositions(expandeeTree)
     } with UnaffiliatedMacroContext {
       val prefix = Expr[Nothing](prefixTree)(TypeTag.Nothing)
       override def toString = "MacroContext(%s@%s +%d)".format(expandee.symbol.name, expandee.pos, enclosingMacros.length - 1 /* exclude myself */)
@@ -371,7 +371,15 @@ trait Macros extends FastTrack with MacroRuntimes with Traces with Helpers {
   def standardMacroArgs(typer: Typer, expandee: Tree): MacroArgs = {
     val macroDef = expandee.symbol
     val paramss = macroDef.paramss
-    val treeInfo.Applied(core, targs, argss) = expandee
+    val treeInfo.Applied(core, targs, maybeNamedArgss) = expandee
+    val argss = map2(maybeNamedArgss, paramss)((args, params) => {
+      if (args.exists(_.isInstanceOf[AssignOrNamedArg])) {
+        val sorted = ListBuffer.fill(params.length)(EmptyTree: Tree)
+        args foreach { case AssignOrNamedArg(Ident(name), arg) => sorted(params.indexWhere(_.name == name)) = arg }
+        sorted.toList
+      } else if (params.length == args.length) args
+      else args ++ List.fill(params.length - args.length)(EmptyTree)
+    })
     val prefix = core match { case Select(qual, _) => qual; case _ => EmptyTree }
     val context = expandee.attachments.get[MacroRuntimeAttachment].flatMap(_.macroContext).getOrElse(macroContext(typer, prefix, expandee))
 
@@ -383,16 +391,11 @@ trait Macros extends FastTrack with MacroRuntimes with Traces with Helpers {
       |paramss: $paramss
     """.trim)
 
-    import typer.TyperErrorGen._
-    val isNullaryArgsEmptyParams = argss.isEmpty && paramss == ListOfNil
-    if (paramss.length < argss.length) MacroTooManyArgumentListsError(expandee)
-    if (paramss.length > argss.length && !isNullaryArgsEmptyParams) MacroTooFewArgumentListsError(expandee)
-
     val macroImplArgs: List[Any] =
       if (fastTrack contains macroDef) {
         // Take a dry run of the fast track implementation
         if (fastTrack(macroDef) validate expandee) argss.flatten
-        else MacroTooFewArgumentListsError(expandee)
+        else typer.TyperErrorGen.MacroFastTrackFailed(expandee)
       }
       else {
         def calculateMacroArgs(binding: MacroImplBinding) = {
@@ -403,14 +406,6 @@ trait Macros extends FastTrack with MacroRuntimes with Traces with Helpers {
           // wrap argss in c.Expr if necessary (i.e. if corresponding macro impl param is of type c.Expr[T])
           // expand varargs (nb! varargs can apply to any parameter section, not necessarily to the last one)
           val trees = map3(argss, paramss, signature)((args, defParams, implParams) => {
-            val isVarargs = isVarArgsList(defParams)
-            if (isVarargs) {
-              if (defParams.length > args.length + 1) MacroTooFewArgumentsError(expandee)
-            } else {
-              if (defParams.length < args.length) MacroTooManyArgumentsError(expandee)
-              if (defParams.length > args.length) MacroTooFewArgumentsError(expandee)
-            }
-
             val wrappedArgs = mapWithIndex(args)((arg, j) => {
               val fingerprint = implParams(min(j, implParams.length - 1))
               fingerprint match {
@@ -421,7 +416,7 @@ trait Macros extends FastTrack with MacroRuntimes with Traces with Helpers {
               }
             })
 
-            if (isVarargs) {
+            if (isVarArgsList(defParams)) {
               val (normal, varargs) = wrappedArgs splitAt (defParams.length - 1)
               normal :+ varargs // pack all varargs into a single Seq argument (varargs Scala style)
             } else wrappedArgs
@@ -529,9 +524,11 @@ trait Macros extends FastTrack with MacroRuntimes with Traces with Helpers {
    *    the expandee with an error marker set   if there has been an error
    */
   abstract class MacroExpander(val typer: Typer, val expandee: Tree) {
+    val symbol = expandee match { case Block(_, expr) => expr.symbol; case tree => tree.symbol }
+
     def onSuccess(expanded: Tree): Tree
     def onFallback(expanded: Tree): Tree
-    def onSuppressed(expandee: Tree): Tree = expandee
+    def onSuppressed(expanded: Tree): Tree = expanded
     def onDelayed(expanded: Tree): Tree = expanded
     def onSkipped(expanded: Tree): Tree = expanded
     def onFailure(expanded: Tree): Tree = { typer.infer.setError(expandee); expandee }
@@ -551,15 +548,15 @@ trait Macros extends FastTrack with MacroRuntimes with Traces with Helpers {
       if (Statistics.canEnable) Statistics.incCounter(macroExpandCount)
       try {
         withInfoLevel(nodePrinters.InfoLevel.Quiet) { // verbose printing might cause recursive macro expansions
-          if (expandee.symbol.isErroneous || (expandee exists (_.isErroneous))) {
-            val reason = if (expandee.symbol.isErroneous) "not found or incompatible macro implementation" else "erroneous arguments"
+          if (symbol.isErroneous || (expandee exists (_.isErroneous)) || (desugared exists (_.isErroneous))) {
+            val reason = if (symbol.isErroneous) "not found or incompatible macro implementation" else "erroneous arguments"
             macroLogVerbose(s"cancelled macro expansion because of $reason: $expandee")
             onFailure(typer.infer.setError(expandee))
           } else try {
             val expanded = {
-              val runtime = macroRuntime(expandee)
-              if (runtime != null) macroExpandWithRuntime(typer, expandee, runtime)
-              else macroExpandWithoutRuntime(typer, expandee)
+              val runtime = macroRuntime(desugared)
+              if (runtime != null) macroExpandWithRuntime(typer, desugared, runtime)
+              else macroExpandWithoutRuntime(typer, desugared)
             }
             expanded match {
               case Success(expanded) =>
@@ -591,7 +588,7 @@ trait Macros extends FastTrack with MacroRuntimes with Traces with Helpers {
   extends MacroExpander(typer, expandee) {
     lazy val innerPt = {
       val tp = if (isNullaryInvocation(expandee)) expandee.tpe.finalResultType else expandee.tpe
-      if (isBlackbox(expandee)) tp
+      if (isBlackbox(symbol)) tp
       else {
         // approximation is necessary for whitebox macros to guide type inference
         // read more in the comments for onDelayed below
@@ -599,6 +596,50 @@ trait Macros extends FastTrack with MacroRuntimes with Traces with Helpers {
         deriveTypeWithWildcards(undetparams)(tp)
       }
     }
+    override protected def expand(desugared: Tree) = {
+      // SI-5940 in order for a macro expansion that involves named or default arguments
+      //         to see the actual prefix and arguments being passed by the user instead of their desugarings
+      //         we need to inline synthetics in case when `fun` is actually a macro
+      //         underlying macro implementation is going to get explicitly passed arguments in correct order
+      //         and the rest (defaults filled in by the vanilla part of `tryNamesDefaults`) will become empty trees
+      //         in order for the macro to be able to account for evaluation order, the original is provided in `c.macroApplication`
+      //         of course, ideally we would like to provide the impl with right-hand sides of those default arguments
+      //         but currently that is flat out impossible because of the difference in scopes
+      //         anyway this is already an improvement over the former status quo when named/default invocations were outright prohibited
+      def undoNamesDefaults(tree: Tree): Tree = {
+        val (qualsym, qual, vdefs0, app @ Applied(_, _, argss)) = tree match {
+          case Block((qualdef @ ValDef(_, name, _, qual)) +: vdefs, app) if name.startsWith(nme.QUAL_PREFIX) => (qualdef.symbol, qual, vdefs, app)
+          case Block(vdefs, app) => (NoSymbol, EmptyTree, vdefs, app)
+          case tree => (NoSymbol, EmptyTree, Nil, tree)
+        }
+        val vdefs = vdefs0.map{ case vdef: ValDef => vdef }
+        def hasNamesDefaults(args: List[Tree]) = {
+          args.exists(arg => isDefaultGetter(arg) || vdefs.exists(_.symbol == arg.symbol))
+        }
+        def undoNamesDefaults(args: List[Tree], depth: Int) = {
+          def extractRhs(vdef: ValDef) = vdef.rhs.changeOwner(vdef.symbol -> typer.context.owner)
+          case class Arg(tree: Tree, ipos: Int, inamed: Int) { val param = app.symbol.paramss(depth)(ipos) }
+          val indexed = args.map(arg => arg -> vdefs.indexWhere(_.symbol == arg.symbol)).zipWithIndex.flatMap({
+            /*    default    */ case ((arg, _), _) if isDefaultGetter(arg) => None
+            /*   positional  */ case ((arg, -1), ipos) => Some(Arg(arg, ipos, -1))
+            /* default+named */ case ((_, inamed), _) if isDefaultGetter(extractRhs(vdefs(inamed))) => None
+            /*     named     */ case ((arg, inamed), ipos) => Some(Arg(extractRhs(vdefs(inamed)), ipos, inamed))
+          })
+          if (indexed.forall(_.inamed == -1)) indexed.map(_.tree)
+          else indexed.sortBy(_.inamed).map(arg => AssignOrNamedArg(Ident(arg.param.name), arg.tree))
+        }
+        def loop(tree: Tree, depth: Int): Tree = tree match {
+          case Apply(fun, args) if hasNamesDefaults(args) => treeCopy.Apply(tree, loop(fun, depth - 1), undoNamesDefaults(args, depth))
+          case Apply(fun, args) => treeCopy.Apply(tree, loop(fun, depth - 1), args)
+          case TypeApply(core, targs) => treeCopy.TypeApply(tree, core, targs)
+          case Select(core, name) if qualsym != NoSymbol && core.symbol == qualsym => treeCopy.Select(tree, qual, name)
+          case core => core
+        }
+        if (app.symbol == null || app.symbol == NoSymbol || app.exists(_.isErroneous)) tree
+        else loop(app, depth = argss.length - 1)
+      }
+      super.expand(undoNamesDefaults(desugared))
+    }
     override def onSuccess(expanded0: Tree) = {
       // prematurely annotate the tree with a macro expansion attachment
       // so that adapt called indirectly by typer.typed knows that it needs to apply the existential fixup
@@ -616,7 +657,7 @@ trait Macros extends FastTrack with MacroRuntimes with Traces with Helpers {
         }
       }
 
-      if (isBlackbox(expandee)) {
+      if (isBlackbox(symbol)) {
         val expanded1 = atPos(enclosingMacroPosition.makeTransparent)(Typed(expanded0, TypeTree(innerPt)))
         typecheck("blackbox typecheck", expanded1, outerPt)
       } else {
@@ -676,7 +717,7 @@ trait Macros extends FastTrack with MacroRuntimes with Traces with Helpers {
       // Thanks to that the materializer can take a look at what's going on and react accordingly.
       val shouldInstantiate = typer.context.undetparams.nonEmpty && !mode.inPolyMode
       if (shouldInstantiate) {
-        if (isBlackbox(expandee)) typer.instantiatePossiblyExpectingUnit(delayed, mode, outerPt)
+        if (isBlackbox(symbol)) typer.instantiatePossiblyExpectingUnit(delayed, mode, outerPt)
         else {
           forced += delayed
           typer.infer.inferExprInstance(delayed, typer.context.extractUndetparams(), outerPt, keepNothings = false)
diff --git a/src/compiler/scala/tools/nsc/typechecker/StdAttachments.scala b/src/compiler/scala/tools/nsc/typechecker/StdAttachments.scala
index 57f27a05fd..1a6d2f0011 100644
--- a/src/compiler/scala/tools/nsc/typechecker/StdAttachments.scala
+++ b/src/compiler/scala/tools/nsc/typechecker/StdAttachments.scala
@@ -85,6 +85,7 @@ trait StdAttachments {
     tree match {
       // see the comment to `isMacroExpansionSuppressed` to learn why we need
       // a special traversal strategy here
+      case Block(_, expr) => unsuppressMacroExpansion(expr)
       case Apply(fn, _) => unsuppressMacroExpansion(fn)
       case TypeApply(fn, _) => unsuppressMacroExpansion(fn)
       case _ => // do nothing
@@ -101,6 +102,8 @@ trait StdAttachments {
         // we have to account for the fact that during typechecking an expandee might become wrapped,
         // i.e. surrounded by an inferred implicit argument application or by an inferred type argument application.
         // in that case the expandee itself will no longer be suppressed and we need to look at the core
+        // upd. we also need to allow for blocks, because that's what names and defaults are often desugared to
+        case Block(_, expr)   => isMacroExpansionSuppressed(expr)
         case Apply(fn, _)     => isMacroExpansionSuppressed(fn)
         case TypeApply(fn, _) => isMacroExpansionSuppressed(fn)
         case _                => false
diff --git a/src/compiler/scala/tools/nsc/typechecker/Typers.scala b/src/compiler/scala/tools/nsc/typechecker/Typers.scala
index 10fe530445..020d26c712 100644
--- a/src/compiler/scala/tools/nsc/typechecker/Typers.scala
+++ b/src/compiler/scala/tools/nsc/typechecker/Typers.scala
@@ -1155,7 +1155,7 @@ trait Typers extends Adaptations with Tags with TypersTracking with PatternTyper
 
         case mt: MethodType if mode.typingExprNotFunNotLhs && mt.isImplicit => // (4.1)
           adaptToImplicitMethod(mt)
-        case mt: MethodType if mode.typingExprNotFunNotLhs && !hasUndetsInMonoMode && !treeInfo.isMacroApplicationOrBlock(tree) =>
+        case mt: MethodType if mode.typingExprNotFunNotLhs && !hasUndetsInMonoMode =>
           instantiateToMethodType(mt)
         case _ =>
           vanillaAdapt(tree)
@@ -3292,12 +3292,6 @@ trait Typers extends Adaptations with Tags with TypersTracking with PatternTyper
            */
           def tryNamesDefaults: Tree = {
             val lencmp = compareLengths(args, formals)
-
-            def checkNotMacro() = {
-              if (treeInfo.isMacroApplication(fun))
-                tryTupleApply orElse duplErrorTree(NamedAndDefaultArgumentsNotSupportedForMacros(tree, fun))
-            }
-
             if (mt.isErroneous) duplErrTree
             else if (mode.inPatternMode) {
               // #2064
@@ -3316,18 +3310,15 @@ trait Typers extends Adaptations with Tags with TypersTracking with PatternTyper
               else if (allArgsArePositional(argPos) && !isNamedApplyBlock(fun)) {
                 // if there's no re-ordering, and fun is not transformed, no need to transform
                 // more than an optimization, e.g. important in "synchronized { x = update-x }"
-                checkNotMacro()
                 doTypedApply(tree, fun, namelessArgs, mode, pt)
               } else {
-                checkNotMacro()
-                transformNamedApplication(Typer.this, mode, pt)(
-                                          treeCopy.Apply(tree, fun, namelessArgs), argPos)
+                unsuppressMacroExpansion(transformNamedApplication(Typer.this, mode, pt)(
+                                         treeCopy.Apply(tree, suppressMacroExpansion(fun), namelessArgs), argPos))
               }
             } else {
               // defaults are needed. they are added to the argument list in named style as
               // calls to the default getters. Example:
               //  foo[Int](a)()  ==>  foo[Int](a)(b = foo$qual.foo$default$2[Int](a))
-              checkNotMacro()
 
               // SI-8111 transformNamedApplication eagerly shuffles around the application to preserve
               //         evaluation order. During this process, it calls `changeOwner` on symbols that
@@ -3350,7 +3341,7 @@ trait Typers extends Adaptations with Tags with TypersTracking with PatternTyper
                 symsOwnedByContextOwner foreach (_.owner = context.owner)
               }
 
-              val fun1 = transformNamedApplication(Typer.this, mode, pt)(fun, x => x)
+              val fun1 = transformNamedApplication(Typer.this, mode, pt)(suppressMacroExpansion(fun), x => x)
               if (fun1.isErroneous) duplErrTree
               else {
                 assert(isNamedApplyBlock(fun1), fun1)
@@ -3376,7 +3367,7 @@ trait Typers extends Adaptations with Tags with TypersTracking with PatternTyper
                   // useful when a default doesn't match parameter type, e.g. def f[T](x:T="a"); f[Int]()
                   val note = "Error occurred in an application involving default arguments."
                   if (!(context.diagnostic contains note)) context.diagnostic = note :: context.diagnostic
-                  doTypedApply(tree, if (blockIsEmpty) fun else fun1, allArgs, mode, pt)
+                  unsuppressMacroExpansion(doTypedApply(tree, if (blockIsEmpty) fun else fun1, allArgs, mode, pt))
                 } else {
                   rollbackNamesDefaultsOwnerChanges()
                   tryTupleApply orElse duplErrorTree(NotEnoughArgsError(tree, fun, missing))
diff --git a/src/reflect/scala/reflect/internal/TreeInfo.scala b/src/reflect/scala/reflect/internal/TreeInfo.scala
index 1eb0743bc8..7ff482d021 100644
--- a/src/reflect/scala/reflect/internal/TreeInfo.scala
+++ b/src/reflect/scala/reflect/internal/TreeInfo.scala
@@ -852,13 +852,8 @@ abstract class TreeInfo {
       case _ => false
     })
 
-  def isMacroApplication(tree: Tree): Boolean = !tree.isDef && {
-    val sym = tree.symbol
-    sym != null && sym.isTermMacro && !sym.isErroneous
-  }
-
-  def isMacroApplicationOrBlock(tree: Tree): Boolean = tree match {
-    case Block(_, expr) => isMacroApplicationOrBlock(expr)
-    case tree => isMacroApplication(tree)
+  def isMacroApplication(tree: Tree): Boolean = tree match {
+    case Block(_, expr) => isMacroApplication(expr)
+    case tree => !tree.isDef && tree.symbol != null && tree.symbol.isTermMacro && !tree.symbol.isErroneous
   }
 }