Highlight comments, remove scanner wrapping syntax highlighter

1 files changed, 119 insertions, 242 deletions

diff --git a/src/dotty/tools/dotc/repl/SyntaxHighlighter.scala b/src/dotty/tools/dotc/repl/SyntaxHighlighter.scala
index ebea3f8b2..1988ea7ad 100644
--- a/src/dotty/tools/dotc/repl/SyntaxHighlighter.scala
+++ b/src/dotty/tools/dotc/repl/SyntaxHighlighter.scala
@@ -2,6 +2,7 @@ package dotty.tools
 package dotc
 package repl
 
+import parsing.Tokens._
 import ammonite.terminal.FilterTools._
 import ammonite.terminal.LazyList._
 import ammonite.terminal.SpecialKeys._
@@ -12,29 +13,29 @@ import scala.collection.mutable.StringBuilder
 
 /** This object provides functions for syntax highlighting in the REPL */
 object SyntaxHighlighting {
+  val NoColor         = Console.RESET
+  val CommentColor    = Console.GREEN
+  val KeywordColor    = Console.CYAN
+  val LiteralColor    = Console.MAGENTA
+  val TypeColor       = Console.GREEN
+  val AnnotationColor = Console.RED
+
   private def none(str: String)       = str
-  private def keyword(str: String)    = Console.CYAN + str + Console.RESET
-  private def typeDef(str: String)    = Console.GREEN + str + Console.RESET
-  private def literal(str: String)    = Console.MAGENTA + str + Console.RESET
-  private def annotation(str: String) = Console.RED + str + Console.RESET
+  private def keyword(str: String)    = KeywordColor + str + NoColor
+  private def typeDef(str: String)    = TypeColor + str + NoColor
+  private def literal(str: String)    = LiteralColor + str + NoColor
+  private def annotation(str: String) = AnnotationColor + str + NoColor
 
-  private val keywords =
-    "abstract"  :: "class"    :: "case"      :: "catch"     :: "def"      ::
-    "do"        :: "extends"  :: "else"      :: "false"     :: "finally"  ::
-    "final"     :: "for"      :: "forSome"   :: "if"        :: "import"   ::
-    "implicit"  :: "lazy"     :: "match"     :: "null"      :: "new"      ::
-    "object"    :: "override" :: "private"   :: "protected" :: "package"  ::
-    "return"    :: "sealed"   :: "super"     :: "true"      :: "trait"    ::
-    "type"      :: "try"      :: "this"      :: "throw"     :: "val"      ::
-    "var"       :: "with"     :: "while"     :: "yield"     :: Nil
+  private val keywords: Seq[String] = for {
+    index <- IF to FORSOME // All alpha keywords
+  } yield tokenString(index)
 
   private val interpolationPrefixes =
-    'A' :: 'B' :: 'C' :: 'D' :: 'E' :: 'F' :: 'G' :: 'H' :: 'I' :: 'J' ::
-    'K' :: 'L' :: 'M' :: 'N' :: 'O' :: 'P' :: 'Q' :: 'R' :: 'S' :: 'T' ::
-    'U' :: 'V' :: 'W' :: 'X' :: 'Y' :: 'Z' :: '$' :: '_' :: 'a' :: 'b' ::
-    'c' :: 'd' :: 'e' :: 'f' :: 'g' :: 'h' :: 'i' :: 'j' :: 'k' :: 'l' ::
-    'm' :: 'n' :: 'o' :: 'p' :: 'q' :: 'r' :: 's' :: 't' :: 'u' :: 'v' ::
-    'w' :: 'x' :: 'y' :: 'z' :: Nil
+    'A' :: 'B' :: 'C' :: 'D' :: 'E' :: 'F' :: 'G' :: 'H' :: 'I' :: 'J' :: 'K' ::
+    'L' :: 'M' :: 'N' :: 'O' :: 'P' :: 'Q' :: 'R' :: 'S' :: 'T' :: 'U' :: 'V' ::
+    'W' :: 'X' :: 'Y' :: 'Z' :: '$' :: '_' :: 'a' :: 'b' :: 'c' :: 'd' :: 'e' ::
+    'f' :: 'g' :: 'h' :: 'i' :: 'j' :: 'k' :: 'l' :: 'm' :: 'n' :: 'o' :: 'p' ::
+    'q' :: 'r' :: 's' :: 't' :: 'u' :: 'v' :: 'w' :: 'x' :: 'y' :: 'z' :: Nil
 
   private val typeEnders =
    '{' :: '}' :: ')' :: '(' :: '=' :: ' ' :: ',' :: '.' :: Nil
@@ -50,7 +51,7 @@ object SyntaxHighlighting {
     @inline def numberStart(c: Char) =
       c.isDigit && (!prev.isLetter || prev == '.' || prev == ' ' || prev == '(' || prev == '\u0000')
 
-    while(iter.hasNext) {
+    while (iter.hasNext) {
       val n = iter.next
       if (interpolationPrefixes.contains(n)) {
         // Interpolation prefixes are a superset of the keyword start chars
@@ -73,6 +74,18 @@ object SyntaxHighlighting {
         }
       } else {
         (n: @switch) match {
+          case '/'  =>
+            if (iter.hasNext) {
+              iter.next match {
+                case '/' => eolComment()
+                case '*' => blockComment()
+                case x => {
+                  newBuf += '/'
+                  val (dup, _) = iter.duplicate
+                  iter = List(x).toIterator ++ dup
+                }
+              }
+            } else newBuf += '/'
           case '='  =>
             append('=', _ == "=>", keyword)
           case '<'  =>
@@ -102,55 +115,97 @@ object SyntaxHighlighting {
       }
     }
 
-    def appendLiteral(delim: Char, nextTwo: Iterator[Char], multiline: Boolean = false) = {
+    def eolComment() = {
+      newBuf append (CommentColor + "//")
+      var curr = '/'
+      while (curr != '\n' && iter.hasNext) {
+        curr = iter.next
+        newBuf += curr
+      }
+      prev = curr
+      newBuf append NoColor
+    }
+
+    def blockComment() = {
+      newBuf append (CommentColor + "/*")
+      var curr = '*'
+      var open = 1
+      while (open > 0 && iter.hasNext) {
+        curr = iter.next
+        newBuf += curr
+
+        if (curr == '*' && iter.hasNext) {
+          curr = iter.next
+          newBuf += curr
+          if (curr == '/') open -= 1
+        } else if (curr == '/' && iter.hasNext) {
+          curr = iter.next
+          newBuf += curr
+          if (curr == '*') open += 1
+        }
+      }
+      prev = curr
+      newBuf append NoColor
+    }
+
+    def appendLiteral(delim: Char, succ: Iterator[Char], multiline: Boolean = false) = {
       var curr: Char      = 0
       var continue        = true
       var closing         = 0
       val inInterpolation = interpolationPrefixes.contains(prev)
-      newBuf append (Console.MAGENTA + delim)
+      newBuf append (LiteralColor + delim)
+
+      def shouldInterpolate =
+        inInterpolation && curr == '$' && prev != '$' && (iter.hasNext || succ.hasNext)
 
+      def interpolate() = {
+        val next: Char = if (succ.hasNext) succ.next else iter.next
+        if (next == '$') {
+          newBuf += curr
+          newBuf += next
+          prev = '$'
+        } else if (next == '{') {
+          newBuf append (KeywordColor + curr)
+          newBuf += next
+          if (iter.hasNext) {
+            var c = iter.next
+            while (iter.hasNext && c != '}') {
+              newBuf += c
+              c = iter.next
+            }
+            newBuf += c
+            newBuf append LiteralColor
+          }
+        } else {
+          newBuf append (KeywordColor + curr)
+          newBuf += next
+          var c: Char = 'a'
+          while (c.isLetterOrDigit && (iter.hasNext || succ.hasNext)) {
+            c = if (succ.hasNext) succ.next else iter.next
+            if (c != '"') newBuf += c
+          }
+          newBuf append LiteralColor
+          if (c == '"') {
+            newBuf += c
+            continue = false
+          }
+        }
+        closing = 0
+      }
 
-      while (continue && (iter.hasNext || nextTwo.hasNext)) {
-        curr = if(nextTwo.hasNext) nextTwo.next else iter.next
-        if (curr == '\\' && (iter.hasNext || nextTwo.hasNext)) {
-          val next = if (nextTwo.hasNext) nextTwo.next else iter.next
-          newBuf append (Console.CYAN + curr)
+      while (continue && (iter.hasNext || succ.hasNext)) {
+        curr = if(succ.hasNext) succ.next else iter.next
+        if (curr == '\\' && (iter.hasNext || succ.hasNext)) {
+          val next = if (succ.hasNext) succ.next else iter.next
+          newBuf append (KeywordColor + curr)
           if (next == 'u') {
             val code = "u" + iter.take(4).mkString
             newBuf append code
           } else newBuf += next
-          newBuf append Console.MAGENTA
-          closing = 0
-        } else if (inInterpolation && curr == '$' && prev != '$' && (iter.hasNext || nextTwo.hasNext)) { //TODO - break me out!
-          val next: Char = if (nextTwo.hasNext) nextTwo.next else iter.next
-          if (next == '$') {
-            newBuf += curr
-            newBuf += next
-            prev = '$'
-          } else if (next == '{') {
-            newBuf append (Console.CYAN + curr)
-            newBuf += next
-            if (iter.hasNext) {
-              var c = iter.next
-              while (iter.hasNext && c != '}') {
-                newBuf += c
-                c = iter.next
-              }
-              newBuf += c
-              newBuf append Console.MAGENTA
-            }
-          } else { //TODO - break me out
-            newBuf append (Console.CYAN + curr)
-            newBuf += next
-            var c: Char = 'a'
-            while (c.isLetterOrDigit && (iter.hasNext || nextTwo.hasNext)) {
-              c = if (nextTwo.hasNext) nextTwo.next else iter.next
-              if (c != '"') newBuf += c
-            }
-            newBuf append Console.MAGENTA
-            if (c == '"') newBuf += c
-          }
+          newBuf append LiteralColor
           closing = 0
+        } else if (shouldInterpolate) {
+          interpolate()
         } else if (curr == delim && multiline) {
           closing += 1
           if (closing == 3) continue = false
@@ -163,8 +218,13 @@ object SyntaxHighlighting {
           closing = 0
         }
       }
-      newBuf append Console.RESET
+      newBuf append NoColor
       prev = curr
+
+      if (succ.hasNext) {
+        val (dup, _) = iter.duplicate
+        iter = succ ++ dup
+      }
     }
 
     def append(c: Char, shouldHL: String => Boolean, highlight: String => String, pre: Iterator[Char] = Iterator.empty) = {
@@ -210,187 +270,4 @@ object SyntaxHighlighting {
 
     newBuf.toVector
   }
-
-  import util.SourceFile
-  import parsing.Scanners.Scanner
-  import dotc.core.Contexts._
-  import dotc.parsing.Tokens._
-  import reporting._
-  def apply(source: SourceFile)(implicit ctx: Context): Vector[Char] = {
-    val freshCtx = ctx.fresh.setReporter(new Reporter {
-      def doReport(d: Diagnostic)(implicit ctx: Context) = ()
-    })
-    val scanner  = new Scanner(source, preserveWhitespace = true)(freshCtx)
-    val buf      = new StringBuilder()
-    var prev     = List(EMPTY)
-
-    var realLength = 0
-    /** `accept` is used to allow for `realLength` to be used to infer "magically
-     *  missing tokens"
-     */
-    def accept(str: String): String = {
-      realLength += str.length
-      str
-    }
-
-    while (scanner.token != EOF) {
-      buf append (scanner.token match {
-        // Whitespace
-        case WHITESPACE => accept({
-          if (prev.head == ERROR) scanner.litBuf.toString
-          else ""
-        } + scanner.strVal)
-
-        // Identifiers
-        case BACKQUOTED_IDENT =>
-          accept(s"""`${scanner.name.show}`""")
-        case id if identifierTokens contains id => {
-          val name = accept(scanner.name.show)
-          if (name.head.isUpper) typeDef(name) else name
-        }
-
-        // Literals
-        case INTERPOLATIONID =>
-          parseInterpStr()
-        case STRINGLIT =>
-          literal(accept(s""""${scanner.strVal}""""))
-        case CHARLIT =>
-          literal(accept(s"""'${scanner.strVal}'"""))
-        case SYMBOLLIT =>
-          accept("'" + scanner.strVal)
-        case lit if literalTokens contains lit =>
-          literal(accept(scanner.strVal))
-
-        // Unclosed literals caught using startedLiteral var
-        case ERROR =>
-          val start = scanner.startedLiteral
-          accept(if (start != null) start else "")
-
-        // Keywords
-        case EQUALS | COLON => accept(scanner.name.show)
-        case k if alphaKeywords.contains(k) || symbolicKeywords.contains(k) =>
-          keyword(accept(scanner.name.show))
-
-        // Other minor tokens (i.e. '{' etc)
-        case EMPTY => ""
-        case XMLSTART => accept("<")
-        case _ => accept(tokenString(scanner.token).replaceAll("\'", ""))
-      })
-      prev = scanner.token :: prev
-      scanner.nextToken()
-    }
-
-    def parseInterpStr(): String = {
-      // print InterpolationID 's' etc
-      val sb = new StringBuilder
-      sb append accept(scanner.name.show)
-      prev = scanner.token :: prev
-      scanner.nextToken()
-
-      /**
-       * The composition of interpolated strings:
-       * s"hello $guy!"
-       * ^ ^^^^^^  ^ ^
-       * |   |     | |
-       * |   |     | |
-       * |   |     | STRINGLIT
-       * |   |     IDENTIFIER
-       * |   STRINGPART
-       * INTERPOLATIONID
-       *
-       * As such, get tokens until EOF or STRINGLIT is encountered
-       */
-      def scan() = scanner.token match {
-        case STRINGPART => {
-          val delim =
-            if (scanner.inMultiLineInterpolation) "\"\"\""
-            else "\""
-
-          if (prev.head == INTERPOLATIONID) literal(accept(delim))
-          else ""
-        } + literal(accept(scanner.strVal))
-
-        case id if identifierTokens contains id => {
-          val name = scanner.name.show
-          // $ symbols are not caught by the scanner, infer them
-          val prefix = if (prev.head == STRINGPART) "$" else ""
-          accept(prefix + name)
-        }
-        case WHITESPACE => accept({
-          // Whitespace occurs in interpolated strings where there
-          // is an error - e.g. unclosed string literal
-          //
-          // Or in blocks i.e. ${...WHITESPACE...}
-          if (prev.head == ERROR) scanner.litBuf.toString
-          else ""
-        } + scanner.strVal)
-        case CHARLIT =>
-          literal(accept(s"""'${scanner.strVal}'"""))
-        case SYMBOLLIT =>
-          accept("'" + scanner.strVal)
-        case lit if literalTokens contains lit  =>
-          literal(accept(scanner.strVal))
-        case LBRACE =>
-          // An `LBRACE` will only occur if it precedes a block, ergo we can
-          // infer "${"
-          accept("${")
-        case RBRACE => accept("}")
-        case ERROR => {
-          // FIXME: the behaviour here is weird, the check on line 329 clashes
-          // with encountering an error in the interpolated string.
-          //
-          // This clause should be the one taking care of the errors!
-          ""
-        }
-        case _ if prev.head == INTERPOLATIONID =>
-          accept("\"")
-        case x => println(s"Unknown symbol: ${scanner.token}"); ???
-      }
-
-      while (scanner.token != EOF && scanner.token != STRINGLIT) {
-        sb append scan
-        prev = scanner.token :: prev
-        scanner.nextToken()
-      }
-
-      val delim =
-        if (scanner.inMultiLineInterpolation) "\"\"\""
-        else "\""
-
-      if (scanner.token == STRINGLIT) {
-        // If the last part of an interpolated string is a literal, it will end
-        // in `STRINGLIT`
-        if (prev.head == INTERPOLATIONID) sb append literal(accept(delim))
-        sb append literal(accept(scanner.strVal + delim))
-      } else if (prev.head == ERROR && prev.tail.head != IDENTIFIER) {
-        // If error entity to occur in an interpolation
-        val litBuf = scanner.litBuf.toString
-        val expectedLength = source.content.length
-        realLength += litBuf.length
-        val str =
-          if (realLength + 4 == expectedLength) "\"\"\"" + litBuf + "$"
-          else if (realLength + 3 == expectedLength) "\"\"\"" + litBuf
-          else if (realLength + 2 == expectedLength) "\"" + litBuf + "$"
-          else "\"" + litBuf
-
-        sb append str
-        prev = -1 :: prev.tail // make sure outer doesn't print this as well
-      } else if (prev.head == ERROR && prev.tail.head == IDENTIFIER) {
-        // If an error is preceeded by an identifier, i.e. later in the interpolation
-        val litBuf = scanner.litBuf.toString
-        val expLen = source.content.length
-
-        val suffix = "" //TODO
-        sb append (litBuf + suffix)
-      } else if (prev.head == IDENTIFIER) {
-        sb append scanner.litBuf.toString
-      } else if (prev.head == INTERPOLATIONID && scanner.token == EOF) {
-        sb append accept(delim)
-        sb append accept(scanner.litBuf.toString)
-      }
-      sb.toString
-    }
-
-    buf.toVector
-  }
 }