From 96cedcdcd82148f091989836eb4959b2c3ec3382 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Tue, 26 Apr 2016 18:23:29 +0200 Subject: Highlight comments, remove scanner wrapping syntax highlighter --- src/dotty/tools/dotc/repl/SyntaxHighlighter.scala | 361 +++++++--------------- 1 file changed, 119 insertions(+), 242 deletions(-) (limited to 'src/dotty/tools/dotc/repl/SyntaxHighlighter.scala') diff --git a/src/dotty/tools/dotc/repl/SyntaxHighlighter.scala b/src/dotty/tools/dotc/repl/SyntaxHighlighter.scala index ebea3f8b2..1988ea7ad 100644 --- a/src/dotty/tools/dotc/repl/SyntaxHighlighter.scala +++ b/src/dotty/tools/dotc/repl/SyntaxHighlighter.scala @@ -2,6 +2,7 @@ package dotty.tools package dotc package repl +import parsing.Tokens._ import ammonite.terminal.FilterTools._ import ammonite.terminal.LazyList._ import ammonite.terminal.SpecialKeys._ @@ -12,29 +13,29 @@ import scala.collection.mutable.StringBuilder /** This object provides functions for syntax highlighting in the REPL */ object SyntaxHighlighting { + val NoColor = Console.RESET + val CommentColor = Console.GREEN + val KeywordColor = Console.CYAN + val LiteralColor = Console.MAGENTA + val TypeColor = Console.GREEN + val AnnotationColor = Console.RED + private def none(str: String) = str - private def keyword(str: String) = Console.CYAN + str + Console.RESET - private def typeDef(str: String) = Console.GREEN + str + Console.RESET - private def literal(str: String) = Console.MAGENTA + str + Console.RESET - private def annotation(str: String) = Console.RED + str + Console.RESET + private def keyword(str: String) = KeywordColor + str + NoColor + private def typeDef(str: String) = TypeColor + str + NoColor + private def literal(str: String) = LiteralColor + str + NoColor + private def annotation(str: String) = AnnotationColor + str + NoColor - private val keywords = - "abstract" :: "class" :: "case" :: "catch" :: "def" :: - "do" :: "extends" :: "else" :: "false" :: "finally" :: - "final" :: "for" :: "forSome" :: "if" :: "import" :: - "implicit" :: "lazy" :: "match" :: "null" :: "new" :: - "object" :: "override" :: "private" :: "protected" :: "package" :: - "return" :: "sealed" :: "super" :: "true" :: "trait" :: - "type" :: "try" :: "this" :: "throw" :: "val" :: - "var" :: "with" :: "while" :: "yield" :: Nil + private val keywords: Seq[String] = for { + index <- IF to FORSOME // All alpha keywords + } yield tokenString(index) private val interpolationPrefixes = - 'A' :: 'B' :: 'C' :: 'D' :: 'E' :: 'F' :: 'G' :: 'H' :: 'I' :: 'J' :: - 'K' :: 'L' :: 'M' :: 'N' :: 'O' :: 'P' :: 'Q' :: 'R' :: 'S' :: 'T' :: - 'U' :: 'V' :: 'W' :: 'X' :: 'Y' :: 'Z' :: '$' :: '_' :: 'a' :: 'b' :: - 'c' :: 'd' :: 'e' :: 'f' :: 'g' :: 'h' :: 'i' :: 'j' :: 'k' :: 'l' :: - 'm' :: 'n' :: 'o' :: 'p' :: 'q' :: 'r' :: 's' :: 't' :: 'u' :: 'v' :: - 'w' :: 'x' :: 'y' :: 'z' :: Nil + 'A' :: 'B' :: 'C' :: 'D' :: 'E' :: 'F' :: 'G' :: 'H' :: 'I' :: 'J' :: 'K' :: + 'L' :: 'M' :: 'N' :: 'O' :: 'P' :: 'Q' :: 'R' :: 'S' :: 'T' :: 'U' :: 'V' :: + 'W' :: 'X' :: 'Y' :: 'Z' :: '$' :: '_' :: 'a' :: 'b' :: 'c' :: 'd' :: 'e' :: + 'f' :: 'g' :: 'h' :: 'i' :: 'j' :: 'k' :: 'l' :: 'm' :: 'n' :: 'o' :: 'p' :: + 'q' :: 'r' :: 's' :: 't' :: 'u' :: 'v' :: 'w' :: 'x' :: 'y' :: 'z' :: Nil private val typeEnders = '{' :: '}' :: ')' :: '(' :: '=' :: ' ' :: ',' :: '.' :: Nil @@ -50,7 +51,7 @@ object SyntaxHighlighting { @inline def numberStart(c: Char) = c.isDigit && (!prev.isLetter || prev == '.' || prev == ' ' || prev == '(' || prev == '\u0000') - while(iter.hasNext) { + while (iter.hasNext) { val n = iter.next if (interpolationPrefixes.contains(n)) { // Interpolation prefixes are a superset of the keyword start chars @@ -73,6 +74,18 @@ object SyntaxHighlighting { } } else { (n: @switch) match { + case '/' => + if (iter.hasNext) { + iter.next match { + case '/' => eolComment() + case '*' => blockComment() + case x => { + newBuf += '/' + val (dup, _) = iter.duplicate + iter = List(x).toIterator ++ dup + } + } + } else newBuf += '/' case '=' => append('=', _ == "=>", keyword) case '<' => @@ -102,55 +115,97 @@ object SyntaxHighlighting { } } - def appendLiteral(delim: Char, nextTwo: Iterator[Char], multiline: Boolean = false) = { + def eolComment() = { + newBuf append (CommentColor + "//") + var curr = '/' + while (curr != '\n' && iter.hasNext) { + curr = iter.next + newBuf += curr + } + prev = curr + newBuf append NoColor + } + + def blockComment() = { + newBuf append (CommentColor + "/*") + var curr = '*' + var open = 1 + while (open > 0 && iter.hasNext) { + curr = iter.next + newBuf += curr + + if (curr == '*' && iter.hasNext) { + curr = iter.next + newBuf += curr + if (curr == '/') open -= 1 + } else if (curr == '/' && iter.hasNext) { + curr = iter.next + newBuf += curr + if (curr == '*') open += 1 + } + } + prev = curr + newBuf append NoColor + } + + def appendLiteral(delim: Char, succ: Iterator[Char], multiline: Boolean = false) = { var curr: Char = 0 var continue = true var closing = 0 val inInterpolation = interpolationPrefixes.contains(prev) - newBuf append (Console.MAGENTA + delim) + newBuf append (LiteralColor + delim) + + def shouldInterpolate = + inInterpolation && curr == '$' && prev != '$' && (iter.hasNext || succ.hasNext) + def interpolate() = { + val next: Char = if (succ.hasNext) succ.next else iter.next + if (next == '$') { + newBuf += curr + newBuf += next + prev = '$' + } else if (next == '{') { + newBuf append (KeywordColor + curr) + newBuf += next + if (iter.hasNext) { + var c = iter.next + while (iter.hasNext && c != '}') { + newBuf += c + c = iter.next + } + newBuf += c + newBuf append LiteralColor + } + } else { + newBuf append (KeywordColor + curr) + newBuf += next + var c: Char = 'a' + while (c.isLetterOrDigit && (iter.hasNext || succ.hasNext)) { + c = if (succ.hasNext) succ.next else iter.next + if (c != '"') newBuf += c + } + newBuf append LiteralColor + if (c == '"') { + newBuf += c + continue = false + } + } + closing = 0 + } - while (continue && (iter.hasNext || nextTwo.hasNext)) { - curr = if(nextTwo.hasNext) nextTwo.next else iter.next - if (curr == '\\' && (iter.hasNext || nextTwo.hasNext)) { - val next = if (nextTwo.hasNext) nextTwo.next else iter.next - newBuf append (Console.CYAN + curr) + while (continue && (iter.hasNext || succ.hasNext)) { + curr = if(succ.hasNext) succ.next else iter.next + if (curr == '\\' && (iter.hasNext || succ.hasNext)) { + val next = if (succ.hasNext) succ.next else iter.next + newBuf append (KeywordColor + curr) if (next == 'u') { val code = "u" + iter.take(4).mkString newBuf append code } else newBuf += next - newBuf append Console.MAGENTA - closing = 0 - } else if (inInterpolation && curr == '$' && prev != '$' && (iter.hasNext || nextTwo.hasNext)) { //TODO - break me out! - val next: Char = if (nextTwo.hasNext) nextTwo.next else iter.next - if (next == '$') { - newBuf += curr - newBuf += next - prev = '$' - } else if (next == '{') { - newBuf append (Console.CYAN + curr) - newBuf += next - if (iter.hasNext) { - var c = iter.next - while (iter.hasNext && c != '}') { - newBuf += c - c = iter.next - } - newBuf += c - newBuf append Console.MAGENTA - } - } else { //TODO - break me out - newBuf append (Console.CYAN + curr) - newBuf += next - var c: Char = 'a' - while (c.isLetterOrDigit && (iter.hasNext || nextTwo.hasNext)) { - c = if (nextTwo.hasNext) nextTwo.next else iter.next - if (c != '"') newBuf += c - } - newBuf append Console.MAGENTA - if (c == '"') newBuf += c - } + newBuf append LiteralColor closing = 0 + } else if (shouldInterpolate) { + interpolate() } else if (curr == delim && multiline) { closing += 1 if (closing == 3) continue = false @@ -163,8 +218,13 @@ object SyntaxHighlighting { closing = 0 } } - newBuf append Console.RESET + newBuf append NoColor prev = curr + + if (succ.hasNext) { + val (dup, _) = iter.duplicate + iter = succ ++ dup + } } def append(c: Char, shouldHL: String => Boolean, highlight: String => String, pre: Iterator[Char] = Iterator.empty) = { @@ -210,187 +270,4 @@ object SyntaxHighlighting { newBuf.toVector } - - import util.SourceFile - import parsing.Scanners.Scanner - import dotc.core.Contexts._ - import dotc.parsing.Tokens._ - import reporting._ - def apply(source: SourceFile)(implicit ctx: Context): Vector[Char] = { - val freshCtx = ctx.fresh.setReporter(new Reporter { - def doReport(d: Diagnostic)(implicit ctx: Context) = () - }) - val scanner = new Scanner(source, preserveWhitespace = true)(freshCtx) - val buf = new StringBuilder() - var prev = List(EMPTY) - - var realLength = 0 - /** `accept` is used to allow for `realLength` to be used to infer "magically - * missing tokens" - */ - def accept(str: String): String = { - realLength += str.length - str - } - - while (scanner.token != EOF) { - buf append (scanner.token match { - // Whitespace - case WHITESPACE => accept({ - if (prev.head == ERROR) scanner.litBuf.toString - else "" - } + scanner.strVal) - - // Identifiers - case BACKQUOTED_IDENT => - accept(s"""`${scanner.name.show}`""") - case id if identifierTokens contains id => { - val name = accept(scanner.name.show) - if (name.head.isUpper) typeDef(name) else name - } - - // Literals - case INTERPOLATIONID => - parseInterpStr() - case STRINGLIT => - literal(accept(s""""${scanner.strVal}"""")) - case CHARLIT => - literal(accept(s"""'${scanner.strVal}'""")) - case SYMBOLLIT => - accept("'" + scanner.strVal) - case lit if literalTokens contains lit => - literal(accept(scanner.strVal)) - - // Unclosed literals caught using startedLiteral var - case ERROR => - val start = scanner.startedLiteral - accept(if (start != null) start else "") - - // Keywords - case EQUALS | COLON => accept(scanner.name.show) - case k if alphaKeywords.contains(k) || symbolicKeywords.contains(k) => - keyword(accept(scanner.name.show)) - - // Other minor tokens (i.e. '{' etc) - case EMPTY => "" - case XMLSTART => accept("<") - case _ => accept(tokenString(scanner.token).replaceAll("\'", "")) - }) - prev = scanner.token :: prev - scanner.nextToken() - } - - def parseInterpStr(): String = { - // print InterpolationID 's' etc - val sb = new StringBuilder - sb append accept(scanner.name.show) - prev = scanner.token :: prev - scanner.nextToken() - - /** - * The composition of interpolated strings: - * s"hello $guy!" - * ^ ^^^^^^ ^ ^ - * | | | | - * | | | | - * | | | STRINGLIT - * | | IDENTIFIER - * | STRINGPART - * INTERPOLATIONID - * - * As such, get tokens until EOF or STRINGLIT is encountered - */ - def scan() = scanner.token match { - case STRINGPART => { - val delim = - if (scanner.inMultiLineInterpolation) "\"\"\"" - else "\"" - - if (prev.head == INTERPOLATIONID) literal(accept(delim)) - else "" - } + literal(accept(scanner.strVal)) - - case id if identifierTokens contains id => { - val name = scanner.name.show - // $ symbols are not caught by the scanner, infer them - val prefix = if (prev.head == STRINGPART) "$" else "" - accept(prefix + name) - } - case WHITESPACE => accept({ - // Whitespace occurs in interpolated strings where there - // is an error - e.g. unclosed string literal - // - // Or in blocks i.e. ${...WHITESPACE...} - if (prev.head == ERROR) scanner.litBuf.toString - else "" - } + scanner.strVal) - case CHARLIT => - literal(accept(s"""'${scanner.strVal}'""")) - case SYMBOLLIT => - accept("'" + scanner.strVal) - case lit if literalTokens contains lit => - literal(accept(scanner.strVal)) - case LBRACE => - // An `LBRACE` will only occur if it precedes a block, ergo we can - // infer "${" - accept("${") - case RBRACE => accept("}") - case ERROR => { - // FIXME: the behaviour here is weird, the check on line 329 clashes - // with encountering an error in the interpolated string. - // - // This clause should be the one taking care of the errors! - "" - } - case _ if prev.head == INTERPOLATIONID => - accept("\"") - case x => println(s"Unknown symbol: ${scanner.token}"); ??? - } - - while (scanner.token != EOF && scanner.token != STRINGLIT) { - sb append scan - prev = scanner.token :: prev - scanner.nextToken() - } - - val delim = - if (scanner.inMultiLineInterpolation) "\"\"\"" - else "\"" - - if (scanner.token == STRINGLIT) { - // If the last part of an interpolated string is a literal, it will end - // in `STRINGLIT` - if (prev.head == INTERPOLATIONID) sb append literal(accept(delim)) - sb append literal(accept(scanner.strVal + delim)) - } else if (prev.head == ERROR && prev.tail.head != IDENTIFIER) { - // If error entity to occur in an interpolation - val litBuf = scanner.litBuf.toString - val expectedLength = source.content.length - realLength += litBuf.length - val str = - if (realLength + 4 == expectedLength) "\"\"\"" + litBuf + "$" - else if (realLength + 3 == expectedLength) "\"\"\"" + litBuf - else if (realLength + 2 == expectedLength) "\"" + litBuf + "$" - else "\"" + litBuf - - sb append str - prev = -1 :: prev.tail // make sure outer doesn't print this as well - } else if (prev.head == ERROR && prev.tail.head == IDENTIFIER) { - // If an error is preceeded by an identifier, i.e. later in the interpolation - val litBuf = scanner.litBuf.toString - val expLen = source.content.length - - val suffix = "" //TODO - sb append (litBuf + suffix) - } else if (prev.head == IDENTIFIER) { - sb append scanner.litBuf.toString - } else if (prev.head == INTERPOLATIONID && scanner.token == EOF) { - sb append accept(delim) - sb append accept(scanner.litBuf.toString) - } - sb.toString - } - - buf.toVector - } } -- cgit v1.2.3