Merge pull request #2830 from densh/topic/stats-parsing-2

updated SI-7331, SI-6843, SI-7731, parser entry point refactoring, assertThrows utility function

6 files changed, 118 insertions, 87 deletions

diff --git a/src/compiler/scala/tools/nsc/ast/parser/Parsers.scala b/src/compiler/scala/tools/nsc/ast/parser/Parsers.scala
index f2d59fcce8..8479df512e 100644
--- a/src/compiler/scala/tools/nsc/ast/parser/Parsers.scala
+++ b/src/compiler/scala/tools/nsc/ast/parser/Parsers.scala
@@ -334,22 +334,27 @@ self =>
 
     def parseStartRule: () => Tree
 
-    /** This is the general parse entry point.
-     */
-    def parse(): Tree = {
-      val t = parseStartRule()
+    def parseRule[T](rule: this.type => T): T = {
+      val t = rule(this)
       accept(EOF)
       t
     }
 
+    /** This is the general parse entry point.
+     */
+    def parse(): Tree = parseRule(_.parseStartRule())
+
+    /** This is alternative entry point for repl, script runner, toolbox and quasiquotes.
+     */
+    def parseStats(): List[Tree] = parseRule(_.templateStats())
+
     /** This is the parse entry point for code which is not self-contained, e.g.
      *  a script which is a series of template statements.  They will be
      *  swaddled in Trees until the AST is equivalent to the one returned
      *  by compilationUnit().
      */
     def scriptBody(): Tree = {
-      val stmts = templateStats()
-      accept(EOF)
+      val stmts = parseStats()
 
       def mainModuleName = newTermName(settings.script.value)
       /* If there is only a single object template in the file and it has a
@@ -563,8 +568,8 @@ self =>
       and
     }
 
-    def expectedMsg(token: Int): String =
-      token2string(token) + " expected but " +token2string(in.token) + " found."
+    def expectedMsgTemplate(exp: String, fnd: String) = s"$exp expected but $fnd found."
+    def expectedMsg(token: Int): String = expectedMsgTemplate(token2string(token), token2string(in.token))
 
     /** Consume one token of the specified type, or signal an error if it is not there. */
     def accept(token: Int): Int = {
@@ -627,6 +632,8 @@ self =>
 
     def isAnnotation: Boolean = in.token == AT
 
+    def isCaseDefStart: Boolean = in.token == CASE
+
     def isLocalModifier: Boolean = in.token match {
       case ABSTRACT | FINAL | SEALED | IMPLICIT | LAZY => true
       case _ => false
@@ -1325,7 +1332,7 @@ self =>
               in.nextToken()
               if (in.token != LBRACE) catchFromExpr()
               else inBracesOrNil {
-                if (in.token == CASE) caseClauses()
+                if (isCaseDefStart) caseClauses()
                 else catchFromExpr()
               }
             }
@@ -1634,7 +1641,7 @@ self =>
      */
     def blockExpr(): Tree = atPos(in.offset) {
       inBraces {
-        if (in.token == CASE) Match(EmptyTree, caseClauses())
+        if (isCaseDefStart) Match(EmptyTree, caseClauses())
         else block()
       }
     }
@@ -2626,7 +2633,7 @@ self =>
           case EQUALS =>
             in.nextToken()
             TypeDef(mods, name, tparams, typ())
-          case SUPERTYPE | SUBTYPE | SEMI | NEWLINE | NEWLINES | COMMA | RBRACE =>
+          case t if t == SUPERTYPE || t == SUBTYPE || t == COMMA || t == RBRACE || isStatSep(t) =>
             TypeDef(mods | Flags.DEFERRED, name, tparams, typeBounds())
           case _ =>
             syntaxErrorOrIncompleteAnd("`=', `>:', or `<:' expected", skipIt = true)(EmptyTree)
@@ -2927,27 +2934,14 @@ self =>
       stats.toList
     }
 
-    /** Informal - for the repl and other direct parser accessors.
-     */
-    def templateStats(): List[Tree] = templateStatSeq(isPre = false)._2 match {
-      case Nil    => EmptyTree.asList
-      case stats  => stats
-    }
-
     /** {{{
-     *  TemplateStatSeq  ::= [id [`:' Type] `=>'] TemplateStat {semi TemplateStat}
-     *  TemplateStat     ::= Import
-     *                     | Annotations Modifiers Def
-     *                     | Annotations Modifiers Dcl
-     *                     | Expr1
-     *                     | super ArgumentExprs {ArgumentExprs}
-     *                     |
+     *  TemplateStatSeq  ::= [id [`:' Type] `=>'] TemplateStats
      *  }}}
      * @param isPre specifies whether in early initializer (true) or not (false)
      */
     def templateStatSeq(isPre : Boolean): (ValDef, List[Tree]) = checkNoEscapingPlaceholders {
       var self: ValDef = emptyValDef
-      val stats = new ListBuffer[Tree]
+      var firstOpt: Option[Tree] = None
       if (isExprIntro) {
         in.flushDoc
         val first = expr(InTemplate) // @S: first statement is potentially converted so cannot be stubbed.
@@ -2964,10 +2958,25 @@ self =>
           }
           in.nextToken()
         } else {
-          stats += first
+          firstOpt = Some(first)
           acceptStatSepOpt()
         }
       }
+      (self, firstOpt ++: templateStats())
+    }
+
+    /** {{{
+     *  TemplateStats    ::= TemplateStat {semi TemplateStat}
+     *  TemplateStat     ::= Import
+     *                     | Annotations Modifiers Def
+     *                     | Annotations Modifiers Dcl
+     *                     | Expr1
+     *                     | super ArgumentExprs {ArgumentExprs}
+     *                     |
+     *  }}}
+     */
+    def templateStats(): List[Tree] = {
+      val stats = new ListBuffer[Tree]
       while (!isStatSeqEnd) {
         if (in.token == IMPORT) {
           in.flushDoc
@@ -2982,7 +2991,14 @@ self =>
         }
         acceptStatSepOpt()
       }
-      (self, stats.toList)
+      stats.toList
+    }
+
+    /** Informal - for the repl and other direct parser accessors.
+     */
+    def templateStatsCompat(): List[Tree] = templateStats() match {
+      case Nil => EmptyTree.asList
+      case stats => stats
     }
 
     /** {{{
@@ -3047,14 +3063,14 @@ self =>
      */
     def blockStatSeq(): List[Tree] = checkNoEscapingPlaceholders {
       val stats = new ListBuffer[Tree]
-      while (!isStatSeqEnd && in.token != CASE) {
+      while (!isStatSeqEnd && !isCaseDefStart) {
         if (in.token == IMPORT) {
           stats ++= importClause()
           acceptStatSep()
         }
         else if (isExprIntro) {
           stats += statement(InBlock)
-          if (in.token != RBRACE && in.token != CASE) acceptStatSep()
+          if (in.token != RBRACE && !isCaseDefStart) acceptStatSep()
         }
         else if (isDefIntro || isLocalModifier || isAnnotation) {
           if (in.token == IMPLICIT) {
diff --git a/src/compiler/scala/tools/nsc/ast/parser/TreeBuilder.scala b/src/compiler/scala/tools/nsc/ast/parser/TreeBuilder.scala
index 666f19851d..ed694023d7 100644
--- a/src/compiler/scala/tools/nsc/ast/parser/TreeBuilder.scala
+++ b/src/compiler/scala/tools/nsc/ast/parser/TreeBuilder.scala
@@ -39,7 +39,7 @@ abstract class TreeBuilder {
    *    x                  becomes      x @ _
    *    x: T               becomes      x @ (_: T)
    */
-  private object patvarTransformer extends Transformer {
+  object patvarTransformer extends Transformer {
     override def transform(tree: Tree): Tree = tree match {
       case Ident(name) if (treeInfo.isVarPattern(tree) && name != nme.WILDCARD) =>
         atPos(tree.pos)(Bind(name, atPos(tree.pos.focus) (Ident(nme.WILDCARD))))
diff --git a/src/compiler/scala/tools/reflect/ToolBoxFactory.scala b/src/compiler/scala/tools/reflect/ToolBoxFactory.scala
index afaca3396c..8d2f200e99 100644
--- a/src/compiler/scala/tools/reflect/ToolBoxFactory.scala
+++ b/src/compiler/scala/tools/reflect/ToolBoxFactory.scala
@@ -269,17 +269,13 @@ abstract class ToolBoxFactory[U <: JavaUniverse](val u: U) { factorySelf =>
       }
 
       def parse(code: String): Tree = {
-        val run = new Run
         reporter.reset()
-        val wrappedCode = "object wrapper {" + EOL + code + EOL + "}"
-        val file = new BatchSourceFile("<toolbox>", wrappedCode)
+        val file = new BatchSourceFile("<toolbox>", code)
         val unit = new CompilationUnit(file)
-        phase = run.parserPhase
-        val parser = newUnitParser(unit)
-        val wrappedTree = parser.parse()
+        val parsed = newUnitParser(unit).parseStats()
         throwIfErrors()
-        val PackageDef(_, List(ModuleDef(_, _, Template(_, _, _ :: parsed)))) = wrappedTree
         parsed match {
+          case Nil => EmptyTree
           case expr :: Nil => expr
           case stats :+ expr => Block(stats, expr)
         }
diff --git a/src/compiler/scala/tools/reflect/quasiquotes/Parsers.scala b/src/compiler/scala/tools/reflect/quasiquotes/Parsers.scala
index 9a6ba56c18..18a806e5ff 100644
--- a/src/compiler/scala/tools/reflect/quasiquotes/Parsers.scala
+++ b/src/compiler/scala/tools/reflect/quasiquotes/Parsers.scala
@@ -17,32 +17,38 @@ trait Parsers { self: Quasiquotes =>
   abstract class Parser extends {
     val global: self.global.type = self.global
   } with ScalaParser {
-    /** Wraps given code to obtain a desired parser mode.
-     *  This way we can just re-use standard parser entry point.
-     */
-    def wrapCode(code: String): String =
-      s"object wrapper { self => $EOL $code $EOL }"
-
-    def unwrapTree(wrappedTree: Tree): Tree = {
-      val PackageDef(_, List(ModuleDef(_, _, Template(_, _, _ :: parsed)))) = wrappedTree
-      parsed match {
-        case tree :: Nil => tree
-        case stats :+ tree => Block(stats, tree)
-      }
-    }
-
     def parse(code: String): Tree = {
       try {
-        val wrapped = wrapCode(code)
-        debug(s"wrapped code\n=${wrapped}\n")
-        val file = new BatchSourceFile(nme.QUASIQUOTE_FILE, wrapped)
-        val tree = new QuasiquoteParser(file).parse()
-        unwrapTree(tree)
+        val file = new BatchSourceFile(nme.QUASIQUOTE_FILE, code)
+        new QuasiquoteParser(file).parseRule(entryPoint)
       } catch {
-        case mi: MalformedInput => c.abort(c.macroApplication.pos, s"syntax error: ${mi.msg}")
+        case mi: MalformedInput => c.abort(correspondingPosition(mi.offset), mi.msg)
+      }
+    }
+
+    def correspondingPosition(offset: Int): Position = {
+      val posMapList = posMap.toList
+      def containsOffset(start: Int, end: Int) = start <= offset && offset <= end
+      def fallbackPosition = posMapList match {
+        case (pos1, (start1, end1)) :: _   if start1 > offset => pos1
+        case _ :+ ((pos2, (start2, end2))) if offset > end2   => pos2.withPoint(pos2.point + (end2 - start2))
       }
+      posMapList.sliding(2).collect {
+        case (pos1, (start1, end1)) :: _                if containsOffset(start1, end1) => (pos1, offset - start1)
+        case (pos1, (_, end1)) :: (_, (start2, _)) :: _ if containsOffset(end1, start2) => (pos1, end1)
+        case _ :: (pos2, (start2, end2)) :: _           if containsOffset(start2, end2) => (pos2, offset - start2)
+      }.map { case (pos, offset) =>
+        pos.withPoint(pos.point + offset)
+      }.toList.headOption.getOrElse(fallbackPosition)
     }
 
+    override def token2string(token: Int): String = token match {
+      case EOF => "end of quote"
+      case _ => super.token2string(token)
+    }
+
+    def entryPoint: QuasiquoteParser => Tree
+
     class QuasiquoteParser(source0: SourceFile) extends SourceFileParser(source0) {
       override val treeBuilder = new ParserTreeBuilder {
         // q"(..$xs)"
@@ -73,9 +79,11 @@ trait Parsers { self: Quasiquotes =>
         } else
           super.caseClause()
 
-      def isHole = isIdent && holeMap.contains(in.name)
+      def isHole: Boolean = isIdent && holeMap.contains(in.name)
 
-      override def isAnnotation: Boolean =  super.isAnnotation || (isHole && lookingAhead { isAnnotation })
+      override def isAnnotation: Boolean = super.isAnnotation || (isHole && lookingAhead { isAnnotation })
+
+      override def isCaseDefStart: Boolean = super.isCaseDefStart || (in.token == EOF)
 
       override def isModifier: Boolean = super.isModifier || (isHole && lookingAhead { isModifier })
 
@@ -85,6 +93,12 @@ trait Parsers { self: Quasiquotes =>
 
       override def isDclIntro: Boolean = super.isDclIntro || (isHole && lookingAhead { isDclIntro })
 
+      override def isStatSep(token: Int) = token == EOF || super.isStatSep(token)
+
+      override def expectedMsg(token: Int): String =
+        if (isHole) expectedMsgTemplate(token2string(token), "splicee")
+        else super.expectedMsg(token)
+
       // $mods def foo
       // $mods T
       override def readAnnots(annot: => Tree): List[Tree] = in.token match {
@@ -101,34 +115,26 @@ trait Parsers { self: Quasiquotes =>
     }
   }
 
-  object TermParser extends Parser
-
-  object CaseParser extends Parser {
-    override def wrapCode(code: String) = super.wrapCode("something match { case " + code + " }")
-
-    override def unwrapTree(wrappedTree: Tree): Tree = {
-      val Match(_, head :: tail) = super.unwrapTree(wrappedTree)
-      if (tail.nonEmpty)
-        c.abort(c.macroApplication.pos, "Can't parse more than one casedef, consider generating a match tree instead")
-      head
+  object TermParser extends Parser {
+    def entryPoint = _.templateStats() match {
+      case Nil => EmptyTree
+      case tree :: Nil => tree
+      case stats :+ tree => Block(stats, tree)
     }
   }
 
-  object PatternParser extends Parser {
-    override def wrapCode(code: String) = super.wrapCode("something match { case " + code + " => }")
-
-    override def unwrapTree(wrappedTree: Tree): Tree = {
-      val Match(_, List(CaseDef(pat, _, _))) = super.unwrapTree(wrappedTree)
-      pat
-    }
+  object TypeParser extends Parser {
+    def entryPoint = _.typ()
   }
 
-  object TypeParser extends Parser {
-    override def wrapCode(code: String) = super.wrapCode("type T = " + code)
+  object CaseParser extends Parser {
+    def entryPoint = _.caseClause()
+  }
 
-    override def unwrapTree(wrappedTree: Tree): Tree = {
-      val TypeDef(_, _, _, rhs) = super.unwrapTree(wrappedTree)
-      rhs
+  object PatternParser extends Parser {
+    def entryPoint = { parser =>
+      val pat = parser.noSeq.pattern1()
+      parser.treeBuilder.patvarTransformer.transform(pat)
     }
   }
 }
\ No newline at end of file
diff --git a/src/compiler/scala/tools/reflect/quasiquotes/Placeholders.scala b/src/compiler/scala/tools/reflect/quasiquotes/Placeholders.scala
index b680c25f76..b3ac1e293a 100644
--- a/src/compiler/scala/tools/reflect/quasiquotes/Placeholders.scala
+++ b/src/compiler/scala/tools/reflect/quasiquotes/Placeholders.scala
@@ -17,18 +17,31 @@ trait Placeholders { self: Quasiquotes =>
   // Step 1: Transform Scala source with holes into vanilla Scala source
 
   lazy val holeMap = new HoleMap()
+  lazy val posMap = mutable.ListMap[Position, (Int, Int)]()
   lazy val code = {
     val sb = new StringBuilder()
     val sessionSuffix = randomUUID().toString.replace("-", "").substring(0, 8) + "$"
 
-    foreach2(args, parts.init) { (tree, p) =>
-      val (part, cardinality) = parseDots(p)
+    def appendPart(value: String, pos: Position) = {
+      val start = sb.length
+      sb.append(value)
+      val end = sb.length
+      posMap += pos -> (start, end)
+    }
+
+    def appendHole(tree: Tree, cardinality: Cardinality) = {
       val placeholderName = c.freshName(TermName(nme.QUASIQUOTE_PREFIX + sessionSuffix))
-      sb.append(part)
       sb.append(placeholderName)
       holeMap(placeholderName) = Hole(tree, cardinality)
     }
-    sb.append(parts.last)
+
+    foreach2(args, parts.init) { case (tree, (p, pos)) =>
+      val (part, cardinality) = parseDots(p)
+      appendPart(part, pos)
+      appendHole(tree, cardinality)
+    }
+    val (p, pos) = parts.last
+    appendPart(p, pos)
 
     sb.toString
   }
diff --git a/src/compiler/scala/tools/reflect/quasiquotes/Quasiquotes.scala b/src/compiler/scala/tools/reflect/quasiquotes/Quasiquotes.scala
index fe954e0bfd..ee99a5e280 100644
--- a/src/compiler/scala/tools/reflect/quasiquotes/Quasiquotes.scala
+++ b/src/compiler/scala/tools/reflect/quasiquotes/Quasiquotes.scala
@@ -17,7 +17,7 @@ abstract class Quasiquotes extends Parsers
   lazy val (universe: Tree, args, parts, parse, reify) = c.macroApplication match {
     case Apply(Select(Select(Apply(Select(universe0, _), List(Apply(_, parts0))), interpolator0), method0), args0) =>
       val parts1 = parts0.map {
-        case Literal(Constant(s: String)) => s
+        case lit @ Literal(Constant(s: String)) => s -> lit.pos
         case part => c.abort(part.pos, "Quasiquotes can only be used with literal strings")
       }
       val reify0 = method0 match {