SI-7854, SI-6768 better parsing/positioning in parser

The parser hole I found while working on the generated positions
serves as the umbrella for a host of improvements. Upgraded
positions assigned during some specific challenging situations mostly
involving the creation of synthetic trees, e.g. for comprehensions
and closures. While doing so improved some error messages.

Eliminated some of the most glaring duplication in the parser.
It's written like there is some payoff associated with being
spectacularly imperative. Not so far.

5 files changed, 283 insertions, 303 deletions

diff --git a/src/compiler/scala/tools/nsc/ast/parser/Parsers.scala b/src/compiler/scala/tools/nsc/ast/parser/Parsers.scala
index 07e24900e9..05ad2dbc57 100644
--- a/src/compiler/scala/tools/nsc/ast/parser/Parsers.scala
+++ b/src/compiler/scala/tools/nsc/ast/parser/Parsers.scala
@@ -303,6 +303,15 @@ self =>
     def o2p(offset: Int): Position
     def r2p(start: Int, mid: Int, end: Int): Position
 
+    /** Creates a range position from the given start offset to
+     *  the value of in.lastOffset.
+     */
+    def rangeSince(start: Int): Position = r2p(start, start, in.lastOffset)
+
+    /** Like in.skipToken, but returns a range position surrounding the skipped token.
+     */
+    def skipTokenRange(): Position = rangeSince(in.skipToken())
+
     /** whether a non-continuable syntax error has been seen */
     private var lastErrorOffset : Int = -1
 
@@ -568,8 +577,9 @@ self =>
       and
     }
 
-    def expectedMsgTemplate(exp: String, fnd: String) = s"$exp expected but $fnd found."
-    def expectedMsg(token: Int): String = expectedMsgTemplate(token2string(token), token2string(in.token))
+    def expectedMsgTemplate(expected: String, found: String): String = s"$expected expected but $found found."
+    def expectedMsg(expected: Int, found: Int): String               = expectedMsgTemplate(token2string(expected), token2string(found))
+    def expectedMsg(token: Int): String                              = expectedMsg(token, in.token)
 
     /** Consume one token of the specified type, or signal an error if it is not there. */
     def accept(token: Int): Int = {
@@ -587,6 +597,10 @@ self =>
       if (in.token == token) in.nextToken()
       offset
     }
+    /** If the given token is available for consumption, consume it and return true.
+     *  Otherwise, do nothing and return false.
+     */
+    def acceptIfPresent(token: Int) = (in.token == token) && { accept(token) ; true }
 
     /** {{{
      *  semi = nl {nl} | `;`
@@ -703,14 +717,24 @@ self =>
 
 /* ---------- TREE CONSTRUCTION ------------------------------------------- */
 
-    def atPos[T <: Tree](offset: Int)(t: T): T                      = atPos(r2p(offset, offset, in.lastOffset max offset))(t)
-    def atPos[T <: Tree](start: Int, point: Int)(t: T): T           = atPos(r2p(start, point, in.lastOffset max start))(t)
+    def atPos[T <: Tree](start: Int)(t: T): T                       = atPos[T](start, start)(t)
+    def atPos[T <: Tree](start: Int, point: Int)(t: T): T           = atPos[T](start, point, in.lastOffset max start)(t)
     def atPos[T <: Tree](start: Int, point: Int, end: Int)(t: T): T = atPos(r2p(start, point, end))(t)
     def atPos[T <: Tree](pos: Position)(t: T): T                    = global.atPos(pos)(t)
 
     def atInPos[T <: Tree](t: T): T  = atPos(o2p(in.offset))(t)
     def setInPos[T <: Tree](t: T): T = t setPos o2p(in.offset)
 
+    /** Use with caution. */
+    def peekahead(): Unit = {
+      in.prev copyFrom in
+      in.nextToken()
+    }
+    def pushback(): Unit = {
+      in.next copyFrom in
+      in copyFrom in.prev
+    }
+
     /** Convert tree to formal parameter list. */
     def convertToParams(tree: Tree): List[ValDef] = tree match {
       case Parens(ts) => ts map convertToParam
@@ -749,10 +773,9 @@ self =>
       if (!sepFirst)
         ts += part
 
-      while (in.token == separator) {
-        in.nextToken()
+      while (acceptIfPresent(separator))
         ts += part
-      }
+
       ts.toList
     }
     @inline final def commaSeparated[T](part: => T): List[T] = tokenSeparated(COMMA, sepFirst = false, part)
@@ -829,10 +852,8 @@ self =>
 
       private def tupleInfixType(start: Int) = {
         in.nextToken()
-        if (in.token == RPAREN) {
-          in.nextToken()
+        if (acceptIfPresent(RPAREN))
           atPos(start, accept(ARROW)) { makeFunctionTypeTree(Nil, typ()) }
-        }
         else {
           val ts = functionTypes()
           accept(RPAREN)
@@ -936,13 +957,8 @@ self =>
       )
 
       def compoundTypeRest(t: Tree): Tree = {
-        val ts = new ListBuffer[Tree] += t
-        while (in.token == WITH) {
-          in.nextToken()
-          ts += annotType()
-        }
+        val types = t :: tokenSeparated(WITH, sepFirst = true, annotType())
         newLineOptWhenFollowedBy(LBRACE)
-        val types         = ts.toList
         val braceOffset   = in.offset
         val hasRefinement = in.token == LBRACE
         val refinements   = if (hasRefinement) refinement() else Nil
@@ -956,7 +972,7 @@ self =>
         }
         // The second case includes an empty refinement - refinements is empty, but
         // it still gets a CompoundTypeTree.
-        ts.toList match {
+        types match {
           case tp :: Nil if !hasRefinement => tp  // single type, no refinement, already positioned
           case tps                         => atPos(t.pos.startOrPoint)(CompoundTypeTree(Template(tps, emptyValDef, refinements)))
         }
@@ -1024,19 +1040,19 @@ self =>
     def path(thisOK: Boolean, typeOK: Boolean): Tree = {
       val start = in.offset
       var t: Tree = null
-      if (in.token == THIS) {
-        in.nextToken()
+      if (acceptIfPresent(THIS)) {
         t = atPos(start) { This(tpnme.EMPTY) }
         if (!thisOK || in.token == DOT) {
           t = selectors(t, typeOK, accept(DOT))
         }
-      } else if (in.token == SUPER) {
-        in.nextToken()
+      }
+      else if (acceptIfPresent(SUPER)) {
         t = atPos(start) { Super(This(tpnme.EMPTY), mixinQualifierOpt()) }
         accept(DOT)
         t = selector(t)
         if (in.token == DOT) t = selectors(t, typeOK, in.skipToken())
-      } else {
+      }
+      else {
         val tok = in.token
         val name = ident()
         t = atPos(start) {
@@ -1045,18 +1061,18 @@ self =>
         }
         if (in.token == DOT) {
           val dotOffset = in.skipToken()
-          if (in.token == THIS) {
-            in.nextToken()
+          if (acceptIfPresent(THIS)) {
             t = atPos(start) { This(name.toTypeName) }
             if (!thisOK || in.token == DOT)
               t = selectors(t, typeOK, accept(DOT))
-          } else if (in.token == SUPER) {
-            in.nextToken()
+          }
+          else if (acceptIfPresent(SUPER)) {
             t = atPos(start) { Super(This(name.toTypeName), mixinQualifierOpt()) }
             accept(DOT)
             t = selector(t)
             if (in.token == DOT) t = selectors(t, typeOK, in.skipToken())
-          } else {
+          }
+          else {
             t = selectors(t, typeOK, dotOffset)
           }
         }
@@ -1065,10 +1081,8 @@ self =>
     }
 
     def selectors(t: Tree, typeOK: Boolean, dotOffset: Int): Tree =
-      if (typeOK && in.token == TYPE) {
-        in.nextToken()
+      if (typeOK && acceptIfPresent(TYPE))
         atPos(t.pos.startOrPoint, dotOffset) { SingletonTypeTree(t) }
-      }
       else {
         val t1 = selector(t)
         if (in.token == DOT) { selectors(t1, typeOK, in.skipToken()) }
@@ -1165,10 +1179,10 @@ self =>
     private def freshPlaceholder(): Tree = {
       val start = in.offset
       val pname = freshName("x$")
-      in.nextToken()
+      accept(USCORE)
       val id = atPos(start)(Ident(pname))
       val param = atPos(id.pos.focus)(gen.mkSyntheticParam(pname.toTermName))
-      placeholderParams = param :: placeholderParams
+      placeholderParams ::= param
       id
     }
 
@@ -1211,7 +1225,7 @@ self =>
 /* ------------- NEW LINES ------------------------------------------------- */
 
     def newLineOpt() {
-      if (in.token == NEWLINE) in.nextToken()
+      acceptIfPresent(NEWLINE)
     }
 
     def newLinesOpt() {
@@ -1235,9 +1249,7 @@ self =>
      *  TypedOpt ::= [`:' Type]
      *  }}}
      */
-    def typedOpt(): Tree =
-      if (in.token == COLON) { in.nextToken(); typ() }
-      else TypeTree()
+    def typedOpt(): Tree = if (acceptIfPresent(COLON)) typ() else TypeTree()
 
     def typeOrInfixType(location: Int): Tree =
       if (location == Local) typ()
@@ -1261,16 +1273,9 @@ self =>
 
 /* ----------- EXPRESSIONS ------------------------------------------------ */
 
-    def condExpr(): Tree = {
-      if (in.token == LPAREN) {
-        in.nextToken()
-        val r = expr()
-        accept(RPAREN)
-        r
-      } else {
-        accept(LPAREN)
-        newLiteral(true)
-      }
+    def condExpr(): Tree = in.token match {
+      case LPAREN => inParens(expr())
+      case _      => syntaxErrorOrIncompleteAnd("parenthesized conditional expression expected", skipIt = false)(newLiteral(true))
     }
 
     /* hook for IDE, unlike expression can be stubbed
@@ -1311,8 +1316,7 @@ self =>
           val cond = condExpr()
           newLinesOpt()
           val thenp = expr()
-          val elsep = if (in.token == ELSE) { in.nextToken(); expr() }
-          else literalUnit
+          val elsep = if (acceptIfPresent(ELSE)) expr() else literalUnit
           If(cond, thenp, elsep)
         }
         parseIf
@@ -1321,23 +1325,19 @@ self =>
           val body = in.token match {
             case LBRACE => inBracesOrUnit(block())
             case LPAREN => inParensOrUnit(expr())
-            case _ => expr()
+            case _      => expr()
           }
           def catchFromExpr() = List(makeCatchFromExpr(expr()))
-          val catches: List[CaseDef] =
-            if (in.token != CATCH) Nil
-            else {
-              in.nextToken()
+          val catches: List[CaseDef] = (
+            if (!acceptIfPresent(CATCH)) Nil else {
               if (in.token != LBRACE) catchFromExpr()
               else inBracesOrNil {
                 if (isCaseDefStart) caseClauses()
                 else catchFromExpr()
               }
             }
-          val finalizer = in.token match {
-            case FINALLY => in.nextToken(); expr()
-            case _ => EmptyTree
-          }
+          )
+          val finalizer = if (acceptIfPresent(FINALLY)) expr() else EmptyTree
           Try(body, catches, finalizer)
         }
         parseTry
@@ -1348,7 +1348,7 @@ self =>
             val cond = condExpr()
             newLinesOpt()
             val body = expr()
-            makeWhile(start, cond, body)
+            makeWhile(cond, body)
           }
         }
         parseWhile
@@ -1367,16 +1367,15 @@ self =>
       case FOR =>
         val start = in.skipToken()
         def parseFor = atPos(start) {
-          val enums =
+          val enums = (
             if (in.token == LBRACE) inBracesOrNil(enumerators())
             else inParensOrNil(enumerators())
+          )
           newLinesOpt()
-          if (in.token == YIELD) {
-            in.nextToken()
+          if (acceptIfPresent(YIELD))
             makeForYield(enums, expr())
-          } else {
+          else
             makeFor(enums, expr())
-          }
         }
         def adjustStart(tree: Tree) =
           if (tree.pos.isRange && start < tree.pos.start)
@@ -1467,9 +1466,8 @@ self =>
       val param0 = convertToParam {
         atPos(in.offset) {
           Ident(ident()) match {
-            case expr if in.token == COLON  =>
-              in.nextToken() ; Typed(expr, typeOrInfixType(location))
-            case expr => expr
+            case expr if acceptIfPresent(COLON) => Typed(expr, typeOrInfixType(location))
+            case expr                           => expr
           }
         }
       }
@@ -1545,32 +1543,28 @@ self =>
      *  }}}
      */
     def simpleExpr(): Tree = {
-      var canApply = true
-      val t =
-        if (isLiteral) literal()
-        else in.token match {
-          case XMLSTART =>
-            xmlLiteral()
-          case IDENTIFIER | BACKQUOTED_IDENT | THIS | SUPER =>
-            path(thisOK = true, typeOK = false)
-          case USCORE =>
-            freshPlaceholder()
-          case LPAREN =>
-            atPos(in.offset)(makeParens(commaSeparated(expr())))
-          case LBRACE =>
-            canApply = false
-            blockExpr()
-          case NEW =>
-            canApply = false
-            val nstart = in.skipToken()
-            val npos = r2p(nstart, nstart, in.lastOffset)
-            val tstart = in.offset
-            val (parents, self, stats) = template()
-            val cpos = r2p(tstart, tstart, in.lastOffset max tstart)
-            gen.mkNew(parents, self, stats, npos, cpos)
-          case _ =>
-            syntaxErrorOrIncompleteAnd("illegal start of simple expression", skipIt = true)(errorTermTree)
-        }
+      val canApply = in.token match {
+        case LBRACE | NEW => false
+        case _            => true
+      }
+      def mkNew(): Tree = {
+        val npos                   = skipTokenRange()
+        val tstart                 = in.offset
+        val (parents, self, stats) = template()
+        val cpos                   = rangeSince(tstart)
+
+        gen.mkNew(parents, self, stats, npos, cpos)
+      }
+      val t = in.token match {
+        case _ if isLiteral                               => literal()
+        case XMLSTART                                     => xmlLiteral()
+        case IDENTIFIER | BACKQUOTED_IDENT | THIS | SUPER => path(thisOK = true, typeOK = false)
+        case USCORE                                       => freshPlaceholder()
+        case LPAREN                                       => atPos(in.offset)(makeParens(commaSeparated(expr())))
+        case LBRACE                                       => blockExpr()
+        case NEW                                          => mkNew()
+        case _                                            => syntaxErrorOrIncompleteAnd("illegal start of simple expression", skipIt = true)(errorTermTree)
+      }
       simpleExprRest(t, canApply = canApply)
     }
 
@@ -1677,9 +1671,9 @@ self =>
      *  Guard ::= if PostfixExpr
      *  }}}
      */
-    def guard(): Tree =
-      if (in.token == IF) { in.nextToken(); stripParens(postfixExpr()) }
-      else EmptyTree
+    def guard(): Tree = if (acceptIfPresent(IF)) guardExpr() else EmptyTree
+
+    def guardExpr(): Tree = stripParens(postfixExpr())
 
     /** {{{
      *  Enumerators ::= Generator {semi Enumerator}
@@ -1704,27 +1698,26 @@ self =>
      *  }}}
      */
     def generator(enums: ListBuffer[Enumerator], eqOK: Boolean) {
-      val start  = in.offset
-      val hasVal = in.token == VAL
-      if (hasVal)
-        in.nextToken()
+      val start      = in.offset
+      val hasVal     = acceptIfPresent(VAL)
+      val pat        = noSeq.pattern1()
+      val point      = in.offset
+      val equalsBody = equalsExprOpt()
+      val hasEq      = !equalsBody.isEmpty
+
+      if (hasVal && !hasEq)
+        syntaxError(in.offset, "val in for comprehension must be followed by assignment")
+      else if (hasEq && !eqOK)
+        syntaxError(point, "for comprehension must start with generator: " + expectedMsg(expected = LARROW, found = EQUALS))
+      else if (hasVal)
+        deprecationWarning(start, "val keyword in for comprehension is deprecated")
+
+      val rhs = equalsBody orElse { accept(LARROW) ; expr() }
 
-      val pat   = noSeq.pattern1()
-      val point = in.offset
-      val hasEq = in.token == EQUALS
-
-      if (hasVal) {
-        if (hasEq) deprecationWarning(in.offset, "val keyword in for comprehension is deprecated")
-        else syntaxError(in.offset, "val in for comprehension must be followed by assignment")
-      }
-
-      if (hasEq && eqOK) in.nextToken()
-      else accept(LARROW)
-      val rhs = expr()
       enums += makeGenerator(r2p(start, point, in.lastOffset max start), pat, hasEq, rhs)
       // why max above? IDE stress tests have shown that lastOffset could be less than start,
       // I guess this happens if instead if a for-expression we sit on a closing paren.
-      while (in.token == IF) enums += makeFilter(in.offset, guard())
+      enums ++= tokenSeparated(IF, sepFirst = true, makeFilter(in.offset, guardExpr()))
     }
 
     def makeFilter(start: Int, tree: Tree) = Filter(r2p(start, tree.pos.point, tree.pos.endOrPoint), tree)
@@ -1832,14 +1825,6 @@ self =>
         var top = simplePattern(badPattern3)
         // after peekahead
         def acceptWildStar() = atPos(top.pos.startOrPoint, in.prev.offset)(Star(stripParens(top)))
-        def peekahead() = {
-          in.prev copyFrom in
-          in.nextToken()
-        }
-        def pushback() = {
-          in.next copyFrom in
-          in copyFrom in.prev
-        }
         // See SI-3189, SI-4832 for motivation. Cf SI-3480 for counter-motivation.
         // TODO: dredge out the remnants of regexp patterns.
         // /{/ peek for _*) or _*} (for xml escape)
@@ -2013,16 +1998,15 @@ self =>
      *  }}}
      */
     def accessQualifierOpt(mods: Modifiers): Modifiers = {
-      var result = mods
-      if (in.token == LBRACKET) {
-        in.nextToken()
-        if (mods.hasAccessBoundary)
-          syntaxError("duplicate private/protected qualifier", skipIt = false)
-        result = if (in.token == THIS) { in.nextToken(); mods | Flags.LOCAL }
-                 else Modifiers(mods.flags, identForType())
-        accept(RBRACKET)
+      def newModifiers(): Modifiers = (
+        if (acceptIfPresent(THIS)) mods | Flags.LOCAL   // private/protected[this]
+        else Modifiers(mods.flags, identForType())      // private/protected[foo]
+      )
+      in.token match {
+        case LBRACKET if mods.hasAccessBoundary => syntaxError("duplicate private/protected qualifier", skipIt = false) ; mods
+        case LBRACKET                           => inBrackets(newModifiers())
+        case _                                  => mods
       }
-      result
     }
 
     private val flagTokens: Map[Int, Long] = Map(
@@ -2160,25 +2144,24 @@ self =>
             }
             paramType()
           }
-        val default =
-          if (in.token == EQUALS) {
-            in.nextToken()
+        val default = (
+          if (acceptIfPresent(EQUALS)) {
             mods |= Flags.DEFAULTPARAM
             expr()
-          } else EmptyTree
+          }
+          else EmptyTree
+        )
         atPos(start, if (name == nme.ERROR) start else nameOffset) {
           ValDef((mods | implicitmod.toLong | bynamemod) withAnnotations annots, name.toTermName, tpt, default)
         }
       }
-      def paramClause(): List[ValDef] = {
-        if (in.token == RPAREN)
-          return Nil
+      def paramClause(): List[ValDef] = in.token match {
+        case RPAREN => Nil
+        case _      =>
+          if (acceptIfPresent(IMPLICIT))
+            implicitmod = Flags.IMPLICIT
 
-        if (in.token == IMPLICIT) {
-          in.nextToken()
-          implicitmod = Flags.IMPLICIT
-        }
-        commaSeparated(param())
+          commaSeparated(param())
       }
       val vds = new ListBuffer[List[ValDef]]
       val start = in.offset
@@ -2186,8 +2169,7 @@ self =>
       if (ofCaseClass && in.token != LPAREN)
         syntaxError(in.lastOffset, "case classes without a parameter list are not allowed;\n"+
                                    "use either case objects or case classes with an explicit `()' as a parameter list.")
-      while (implicitmod == 0 && in.token == LPAREN) {
-        in.nextToken()
+      while (implicitmod == 0 && acceptIfPresent(LPAREN)) {
         vds += paramClause()
         accept(RPAREN)
         caseParam = false
@@ -2211,18 +2193,16 @@ self =>
     def paramType(): Tree = paramType(useStartAsPosition = false)
     def paramType(useStartAsPosition: Boolean): Tree = {
       val start = in.offset
-      in.token match {
-        case ARROW  =>
+      if (acceptIfPresent(ARROW))
+        atPos(start)(byNameApplication(typ()))
+      else {
+        val t = typ()
+        if (isRawStar) {
           in.nextToken()
-          atPos(start)(byNameApplication(typ()))
-        case _      =>
-          val t = typ()
-          if (isRawStar) {
-            in.nextToken()
-            if (useStartAsPosition) atPos(start)(repeatedApplication(t))
-            else atPos(t.pos.startOrPoint, t.pos.point)(repeatedApplication(t))
-          }
-          else t
+          if (useStartAsPosition) atPos(start)(repeatedApplication(t))
+          else atPos(t.pos.startOrPoint, t.pos.point)(repeatedApplication(t))
+        }
+        else t
       }
     }
 
@@ -2257,14 +2237,13 @@ self =>
         }
         if (contextBoundBuf ne null) {
           while (in.token == VIEWBOUND) {
-            contextBoundBuf += atPos(in.skipToken()) {
-              makeFunctionTypeTree(List(Ident(pname)), typ())
-            }
+            contextBoundBuf += atPos(in.skipToken())(makeFunctionTypeTree(List(Ident(pname)), typ()))
           }
           while (in.token == COLON) {
-            contextBoundBuf += atPos(in.skipToken()) {
-              AppliedTypeTree(typ(), List(Ident(pname)))
-            }
+            val start   = in.skipToken()
+            val tycon   = typ()
+            val applied = atPos(tycon.pos withStart start)(AppliedTypeTree(tycon, Ident(pname) :: Nil))
+            contextBoundBuf += applied
           }
         }
         param
@@ -2290,7 +2269,7 @@ self =>
         t setPos o2p(in.offset)
     }
 
-    def bound(tok: Int): Tree = if (in.token == tok) { in.nextToken(); typ() } else EmptyTree
+    def bound(tok: Int): Tree = if (acceptIfPresent(tok)) typ() else EmptyTree
 
 /* -------- DEFS ------------------------------------------- */
 
@@ -2302,11 +2281,10 @@ self =>
     def importClause(): List[Tree] = {
       val offset = accept(IMPORT)
       commaSeparated(importExpr()) match {
-        case Nil => Nil
+        case Nil       => Nil
         case t :: rest =>
           // The first import should start at the position of the keyword.
-          t.setPos(t.pos.withStart(offset))
-          t :: rest
+          (t setPos (t.pos withStart offset)) :: rest
       }
     }
 
@@ -2335,12 +2313,9 @@ self =>
           case _        =>
             val nameOffset = in.offset
             val name = ident()
-            if (in.token == DOT) {
+            if (acceptIfPresent(DOT))
               // import foo.bar.ident.<unknown> and so create a select node and recurse.
-              val t = atPos(start, if (name == nme.ERROR) in.offset else nameOffset)(Select(expr, name))
-              in.nextToken()
-              return loop(t)
-            }
+              return loop(atPos(start, if (name == nme.ERROR) in.offset else nameOffset)(Select(expr, name)))
             // import foo.bar.Baz;
             else List(makeImportSelector(name, nameOffset))
         }
@@ -2371,30 +2346,27 @@ self =>
       selectors
     }
 
-    def wildcardOrIdent() = {
-      if (in.token == USCORE) { in.nextToken() ; nme.WILDCARD }
-      else ident()
-    }
+    def wildcardOrIdent() = if (acceptIfPresent(USCORE)) nme.WILDCARD else ident()
 
     /** {{{
      *  ImportSelector ::= Id [`=>' Id | `=>' `_']
      *  }}}
      */
     def importSelector(): ImportSelector = {
-      val start        = in.offset
-      val name         = wildcardOrIdent()
-      var renameOffset = -1
-      val rename       = in.token match {
-        case ARROW    =>
-          in.nextToken()
-          renameOffset = in.offset
-          wildcardOrIdent()
-        case _ if name == nme.WILDCARD  => null
-        case _ =>
-          renameOffset = start
-          name
-      }
-      ImportSelector(name, start, rename, renameOffset)
+      val start = in.offset
+      val name  = wildcardOrIdent()
+
+      // The first case is overly cleverly using named arguments to reverse the
+      // positions of the last two parameters, because reading the rename will
+      // move the value of in.offset. Hey, I didn't invent side effects, I too am
+      // a victim here. I can't find a single place where the rename position
+      // is used anyway.
+      if (acceptIfPresent(ARROW))
+        ImportSelector(name, start, renamePos = in.offset, rename = wildcardOrIdent())
+      else if (name == nme.WILDCARD)
+        ImportSelector(name, start, null, -1)
+      else
+        ImportSelector(name, start, name, start)
     }
 
     /** {{{
@@ -2433,6 +2405,8 @@ self =>
       defOrDcl(caseAwareTokenOffset, modifiers() withAnnotations annots)
     }
 
+    def equalsExprOpt(): Tree = if (acceptIfPresent(EQUALS)) expr() else EmptyTree
+
     /** {{{
      *  PatDef ::= Pattern2 {`,' Pattern2} [`:' Type] `=' Expr
      *  ValDcl ::= Id {`,' Id} `:' Type
@@ -2440,45 +2414,55 @@ self =>
      *  }}}
      */
     def patDefOrDcl(pos : Int, mods: Modifiers): List[Tree] = {
-      var newmods = mods
       in.nextToken()
-      val lhs = commaSeparated(stripParens(noSeq.pattern2()))
-      val tp = typedOpt()
-      val rhs =
-        if (tp.isEmpty || in.token == EQUALS) {
-          accept(EQUALS)
-          if (!tp.isEmpty && newmods.isMutable &&
-              (lhs.toList forall (_.isInstanceOf[Ident])) && in.token == USCORE) {
-            in.nextToken()
-            newmods = newmods | Flags.DEFAULTINIT
-            EmptyTree
-          } else {
-            expr()
-          }
-        } else {
-          newmods = newmods | Flags.DEFERRED
-          EmptyTree
-        }
-      def mkDefs(p: Tree, tp: Tree, rhs: Tree): List[Tree] = {
-        //Console.println("DEBUG: p = "+p.toString()); // DEBUG
-        val trees =
-          makePatDef(newmods,
-                     if (tp.isEmpty) p
-                     else Typed(p, tp) setPos (p.pos union tp.pos),
-                     rhs)
-        if (newmods.isDeferred) {
-          trees match {
-            case List(ValDef(_, _, _, EmptyTree)) =>
-              if (mods.isLazy) syntaxError(p.pos, "lazy values may not be abstract", skipIt = false)
-            case _ => syntaxError(p.pos, "pattern definition may not be abstract", skipIt = false)
-          }
+
+      val lhses         = commaSeparated(stripParens(noSeq.pattern2()))
+      val lhs           = lhses.last
+      val tpt           = typedOpt()
+      val ascriptedLhs  = if (tpt.isEmpty) lhs else atPos(lhs.pos union tpt.pos)(Typed(lhs, tpt))
+      val hasEq         = acceptIfPresent(EQUALS)
+      // SI-7854 an underscore following the equals doesn't necessarily mean default initialization.
+      val isDefaultInit = hasEq && in.token == USCORE && {
+        peekahead()
+        isStatSep || isStatSeqEnd || { pushback() ; false }
+      }
+      val rhs       = if (hasEq && !isDefaultInit) expr() else EmptyTree
+      def allIdents = lhses forall (_.isInstanceOf[Ident])
+
+      def defaultInitFlag(): Long = {
+        if (!allIdents)
+          syntaxError(lhs.pos, "pattern definition is ineligible for default initialization", skipIt = false)
+        else if (!mods.isMutable)
+          syntaxError(lhs.pos, "only vars are eligible for default initialization", skipIt = false)
+        else if (tpt.isEmpty)
+          syntaxError(lhs.pos, "an explicit type is required for default initialization", skipIt = false)
+
+        Flags.DEFAULTINIT
+      }
+      def deferredFlag(): Long = {
+        if (mods.isLazy) // e.g. lazy val foo: Int
+          syntaxError(lhs.pos, "lazy values may not be abstract", skipIt = false)
+        else if (!allIdents) // e.g. val Some(x)
+          syntaxError(lhs.pos, "pattern definition may not be abstract", skipIt = false)
+
+        Flags.DEFERRED
+      }
+      val newmods = mods | (
+        if (isDefaultInit) defaultInitFlag()
+        else if (rhs.isEmpty) deferredFlag()
+        else 0L
+      )
+
+      def makeValDefs(decl: Tree): List[Tree] = {
+        val newTpt = if (tpt.isEmpty) decl else Typed(decl, tpt.duplicate setPos tpt.pos.focus) setPos decl.pos.focus
+        makePatDef(newmods, newTpt, rhs.duplicate setPos rhs.pos.focus) match {
+          case tree :: Nil => (tree setPos decl.pos) :: Nil
+          case trees       => trees map (_ setPos decl.pos.focus)
         }
-        trees
       }
-      val trees = (lhs.toList.init flatMap (mkDefs(_, tp.duplicate, rhs.duplicate))) ::: mkDefs(lhs.last, tp, rhs)
-      val hd = trees.head
-      hd setPos hd.pos.withStart(pos)
-      ensureNonOverlapping(hd, trees.tail)
+
+      val trees = (lhses.init flatMap makeValDefs) ::: makePatDef(newmods, ascriptedLhs, rhs)
+      ensureNonOverlapping(trees.last, trees.init)
       trees
     }
 
@@ -2523,7 +2507,8 @@ self =>
       in.nextToken()
       if (in.token == THIS) {
         atPos(start, in.skipToken()) {
-          val vparamss = paramClauses(nme.CONSTRUCTOR, classContextBounds map (_.duplicate), ofCaseClass = false)
+          val cbounds = classContextBounds map (_.duplicate)
+          val vparamss = paramClauses(nme.CONSTRUCTOR, cbounds, ofCaseClass = false)
           newLineOptWhenFollowedBy(LBRACE)
           val rhs = in.token match {
             case LBRACE   => atPos(in.offset) { constrBlock(vparamss) }
@@ -2547,7 +2532,8 @@ self =>
         // i.e. (B[T] or T => B)
         val contextBoundBuf = new ListBuffer[Tree]
         val tparams = typeParamClauseOpt(name, contextBoundBuf)
-        val vparamss = paramClauses(name, contextBoundBuf.toList, ofCaseClass = false)
+        val cbounds = contextBoundBuf.toList
+        val vparamss = paramClauses(name, cbounds, ofCaseClass = false)
         newLineOptWhenFollowedBy(LBRACE)
         var restype = fromWithinReturnType(typedOpt())
         val rhs =
@@ -2707,8 +2693,9 @@ self =>
           val result = gen.mkClassDef(mods1, name, tparams, template)
           // Context bounds generate implicit parameters (part of the template) with types
           // from tparams: we need to ensure these don't overlap
-          if (!classContextBounds.isEmpty)
+          if (classContextBounds.nonEmpty)
             ensureNonOverlapping(template, tparams)
+
           result
         }
       }
@@ -2770,18 +2757,15 @@ self =>
      *  }}}
      */
     def templateParents(): List[Tree] = {
-      val parents = new ListBuffer[Tree]
-      def readAppliedParent() = {
+      def readAppliedParent(): Tree = {
         val start = in.offset
         val parent = startAnnotType()
-        parents += (in.token match {
+        in.token match {
           case LPAREN => atPos(start)((parent /: multipleArgumentExprs())(Apply.apply))
           case _      => parent
-        })
+        }
       }
-      readAppliedParent()
-      while (in.token == WITH) { in.nextToken(); readAppliedParent() }
-      parents.toList
+      tokenSeparated(WITH, sepFirst = false, readAppliedParent())
     }
 
     /** {{{
@@ -2815,9 +2799,9 @@ self =>
     def ensureEarlyDef(tree: Tree): Tree = tree match {
       case vdef @ ValDef(mods, _, _, _) if !mods.isDeferred =>
         copyValDef(vdef)(mods = mods | Flags.PRESUPER)
-      case tdef @ TypeDef(mods, name, tparams, rhs) =>
+      case tdef @ TypeDef(mods, _, _, _) =>
         deprecationWarning(tdef.pos.point, "early type members are deprecated. Move them to the regular body: the semantics are the same.")
-        treeCopy.TypeDef(tdef, mods | Flags.PRESUPER, name, tparams, rhs)
+        copyTypeDef(tdef)(mods = mods | Flags.PRESUPER)
       case docdef @ DocDef(comm, rhs) =>
         treeCopy.DocDef(docdef, comm, rhs)
       case stat if !stat.isEmpty =>
@@ -2852,7 +2836,6 @@ self =>
       )
       val parentPos = o2p(in.offset)
       val tstart1 = if (body.isEmpty && in.lastOffset < tstart) in.lastOffset else tstart
-
       atPos(tstart1) {
         // Exclude only the 9 primitives plus AnyVal.
         if (inScalaRootPackage && ScalaValueClassNames.contains(name))
@@ -3103,10 +3086,9 @@ self =>
     def compilationUnit(): PackageDef = checkNoEscapingPlaceholders {
       def topstats(): List[Tree] = {
         val ts = new ListBuffer[Tree]
-        while (in.token == SEMI) in.nextToken()
+        while (acceptIfPresent(SEMI)) ()
         val start = in.offset
-        if (in.token == PACKAGE) {
-          in.nextToken()
+        if (acceptIfPresent(PACKAGE)) {
           if (in.token == OBJECT) {
             // TODO - this next line is supposed to be
             //    ts += packageObjectDef(start)
diff --git a/src/compiler/scala/tools/nsc/ast/parser/Scanners.scala b/src/compiler/scala/tools/nsc/ast/parser/Scanners.scala
index 6957f85689..5ef40923b4 100644
--- a/src/compiler/scala/tools/nsc/ast/parser/Scanners.scala
+++ b/src/compiler/scala/tools/nsc/ast/parser/Scanners.scala
@@ -52,6 +52,15 @@ trait Scanners extends ScannersCommon {
   type Offset = Int
 
   trait TokenData extends CommonTokenData {
+    override def toString = s"""
+      |TokenData(
+      |   token  $token  ${token2string(token)}
+      |  offset  $offset
+      |    last  $lastOffset
+      |    name  $name
+      |  strVal  $strVal
+      |    base  $base
+      |)""".stripMargin.trim
 
     /** the next token */
     var token: Int = EMPTY
diff --git a/src/compiler/scala/tools/nsc/ast/parser/TreeBuilder.scala b/src/compiler/scala/tools/nsc/ast/parser/TreeBuilder.scala
index 91ff530e05..976e578afd 100644
--- a/src/compiler/scala/tools/nsc/ast/parser/TreeBuilder.scala
+++ b/src/compiler/scala/tools/nsc/ast/parser/TreeBuilder.scala
@@ -206,22 +206,21 @@ abstract class TreeBuilder {
   }
 
   /** Create tree representing a while loop */
-  def makeWhile(startPos: Int, cond: Tree, body: Tree): Tree = {
-    val lname = freshTermName(nme.WHILE_PREFIX)
-    def default = wrappingPos(List(cond, body)) match {
-      case p if p.isDefined => p.endOrPoint
-      case _                => startPos
-    }
-    val continu = atPos(o2p(body.pos pointOrElse default)) { Apply(Ident(lname), Nil) }
-    val rhs = If(cond, Block(List(body), continu), Literal(Constant(())))
-    LabelDef(lname, Nil, rhs)
+  def makeWhile(cond: Tree, body: Tree): Tree = {
+    val lname   = freshTermName(nme.WHILE_PREFIX)
+    val continu = atPos(cond.pos.focus)(Apply(Ident(lname), Nil))
+    val rhs     = atPos(cond.pos union body.pos)(If(cond, atPos(body.pos)(Block(body :: Nil, continu)), Literal(Constant(()))))
+
+    atPos(rhs.pos)(LabelDef(lname, Nil, rhs))
   }
 
   /** Create tree representing a do-while loop */
   def makeDoWhile(lname: TermName, body: Tree, cond: Tree): Tree = {
-    val continu = Apply(Ident(lname), Nil)
-    val rhs = Block(List(body), If(cond, continu, Literal(Constant(()))))
-    LabelDef(lname, Nil, rhs)
+    val continu   = atPos(cond.pos.focus)(Apply(Ident(lname), Nil))
+    val condition = atPos(cond.pos)(If(cond, continu, Literal(Constant(()))))
+    val rhs       = atPos(cond.pos union body.pos)(Block(body :: Nil, condition))
+
+    atPos(rhs.pos)(LabelDef(lname, Nil, rhs))
   }
 
   /** Create block of statements `stats`  */
@@ -313,19 +312,13 @@ abstract class TreeBuilder {
      * The closure is assigned a transparent position with the point at pos.point and
      * the limits given by pat and body.
      */
-    def makeClosure(pos: Position, pat: Tree, body: Tree): Tree = {
-      def splitpos = wrappingPos(List(pat, body)).withPoint(pos.point).makeTransparent
-      matchVarPattern(pat) match {
-        case Some((name, tpt)) =>
-          Function(
-            List(atPos(pat.pos) { ValDef(Modifiers(PARAM), name.toTermName, tpt, EmptyTree) }),
-            body) setPos splitpos
-        case None =>
-          atPos(splitpos) {
-            makeVisitor(List(CaseDef(pat, EmptyTree, body)), checkExhaustive = false)
-          }
+    def makeClosure(pos: Position, pat: Tree, body: Tree): Tree =
+      atPos((pos union pat.pos union body.pos).makeTransparent) {
+        matchVarPattern(pat) match {
+          case Some((name, tpt)) => Function(atPos(pat.pos)(ValDef(Modifiers(PARAM), name.toTermName, tpt, EmptyTree)) :: Nil, body)
+          case None              => makeVisitor(CaseDef(pat, body) :: Nil, checkExhaustive = false)
+        }
       }
-    }
 
     /* Make an application  qual.meth(pat => body) positioned at `pos`.
      */
@@ -503,7 +496,7 @@ abstract class TreeBuilder {
                      tmp, TypeTree(), matchExpr)
             }
           var cnt = 0
-          val restDefs = for ((vname, tpt, pos) <- vars) yield atPos(pos) {
+          val restDefs = for ((vname, tpt, pos) <- vars) yield atPos(pos.focus) {
             cnt += 1
             ValDef(mods, vname.toTermName, tpt, Select(Ident(tmp), newTermName("_" + cnt)))
           }
diff --git a/src/reflect/scala/reflect/internal/TreeGen.scala b/src/reflect/scala/reflect/internal/TreeGen.scala
index 72e1ee107b..a8af3e0f0e 100644
--- a/src/reflect/scala/reflect/internal/TreeGen.scala
+++ b/src/reflect/scala/reflect/internal/TreeGen.scala
@@ -280,7 +280,7 @@ abstract class TreeGen extends macros.TreeBuilder {
 
   /** Builds a tuple */
   def mkTuple(elems: List[Tree]): Tree =
-    if (elems.isEmpty) Literal(Constant(()))
+    if (elems.isEmpty) mkUnit()
     else Apply(
       Select(mkAttributedRef(TupleClass(elems.length).caseModule), nme.apply),
       elems)
@@ -326,8 +326,7 @@ abstract class TreeGen extends macros.TreeBuilder {
    *    body
    *  }
    */
-  def mkTemplate(parents: List[Tree], self: ValDef, constrMods: Modifiers,
-                 vparamss: List[List[ValDef]], body: List[Tree], superPos: Position = NoPosition): Template = {
+  def mkTemplate(parents: List[Tree], self: ValDef, constrMods: Modifiers, vparamss: List[List[ValDef]], body: List[Tree], superPos: Position = NoPosition): Template = {
     /* Add constructor to template */
 
     // create parameters for <init> as synthetic trees.
@@ -398,51 +397,41 @@ abstract class TreeGen extends macros.TreeBuilder {
    *  @param npos  the position of the new
    *  @param cpos  the position of the anonymous class starting with parents
    */
-  def mkNew(parents: List[Tree], self: ValDef, stats: List[Tree],
-            npos: Position, cpos: Position): Tree =
-    if (parents.isEmpty)
-      mkNew(List(scalaAnyRefConstr), self, stats, npos, cpos)
-    else if (parents.tail.isEmpty && stats.isEmpty) {
-      // `Parsers.template` no longer differentiates tpts and their argss
-      // e.g. `C()` will be represented as a single tree Apply(Ident(C), Nil)
-      // instead of parents = Ident(C), argss = Nil as before
-      // this change works great for things that are actually templates
-      // but in this degenerate case we need to perform postprocessing
-      val app = treeInfo.dissectApplied(parents.head)
-      atPos(npos union cpos) { New(app.callee, app.argss) }
-    } else {
-      val x = tpnme.ANON_CLASS_NAME
-      atPos(npos union cpos) {
-        Block(
-          List(
-            atPos(cpos) {
-              ClassDef(
-                Modifiers(FINAL), x, Nil,
-                mkTemplate(parents, self, NoMods, ListOfNil, stats, cpos.focus))
-            }),
-          atPos(npos) {
-            New(
-              Ident(x) setPos npos.focus,
-              Nil)
-          }
-        )
-      }
+  def mkNew(parents: List[Tree], self: ValDef, stats: List[Tree], npos: Position, cpos: Position): Tree = {
+    def enclosingPos = wrappingPos(cpos, parents ::: List(self) ::: stats)
+    def upos         = cpos union npos
+    def anonTemplate = atPos(cpos)(mkTemplate(parents, self, NoMods, ListOfNil, stats, cpos))
+    def anonClass    = atPos(anonTemplate.pos.makeTransparent)(ClassDef(Modifiers(FINAL), tpnme.ANON_CLASS_NAME, Nil, anonTemplate))
+    def anonNew      = atPos(npos)(New(Ident(tpnme.ANON_CLASS_NAME) setPos cpos.focus, Nil))
+
+    // `Parsers.template` no longer differentiates tpts and their argss
+    // e.g. `C()` will be represented as a single tree Apply(Ident(C), Nil)
+    // instead of parents = Ident(C), argss = Nil as before
+    // this change works great for things that are actually templates
+    // but in this degenerate case we need to perform postprocessing
+    parents match {
+      case Nil                                                         => mkNew(List(scalaAnyRefConstr), self, stats, npos, cpos)
+      case treeInfo.AppliedArgs(callee, argss) :: Nil if stats.isEmpty => atPos(upos)(New(callee, argss))
+      case _                                                           => atPos(upos)(mkBlock(anonClass :: anonNew :: Nil))
     }
+  }
 
   /** Create a tree representing the function type (argtpes) => restpe */
   def mkFunctionTypeTree(argtpes: List[Tree], restpe: Tree): Tree =
     AppliedTypeTree(rootScalaDot(newTypeName("Function" + argtpes.length)), argtpes ::: List(restpe))
 
+  def mkUnit() = Literal(Constant(()))
+
   /** Create block of statements `stats`  */
-  def mkBlock(stats: List[Tree]): Tree =
-    if (stats.isEmpty) Literal(Constant(()))
-    else if (!stats.last.isTerm) Block(stats, Literal(Constant(())))
-    else if (stats.length == 1) stats.head
-    else Block(stats.init, stats.last)
+  def mkBlock(stats: List[Tree]): Tree = stats match {
+    case stats if stats.isEmpty || !stats.last.isTerm => mkBlock(stats :+ mkUnit())
+    case stat :: Nil                                  => stat
+    case stats                                        => Block(stats.init, stats.last)
+  }
 
   def mkTreeOrBlock(stats: List[Tree]) = stats match {
-    case Nil => EmptyTree
+    case Nil         => EmptyTree
     case head :: Nil => head
-    case _ => gen.mkBlock(stats)
+    case _           => gen.mkBlock(stats)
   }
 }
diff --git a/src/reflect/scala/reflect/internal/TreeInfo.scala b/src/reflect/scala/reflect/internal/TreeInfo.scala
index 34fe0afb1a..1b763b8632 100644
--- a/src/reflect/scala/reflect/internal/TreeInfo.scala
+++ b/src/reflect/scala/reflect/internal/TreeInfo.scala
@@ -773,6 +773,13 @@ abstract class TreeInfo {
       unapply(dissectApplied(tree))
   }
 
+  object AppliedArgs {
+    def unapply(tree: Tree): Some[(Tree, List[List[Tree]])] = tree match {
+      case Apply(AppliedArgs(fn, argss), args) => Some((fn, argss :+ args))
+      case _                                   => Some((tree, Nil))
+    }
+  }
+
   /** Locates the synthetic Apply node corresponding to an extractor's call to
    *  unapply (unwrapping nested Applies) and returns the fun part of that Apply.
    */