From f010c62b8dc8baaaa9b87784d8664997bee95fa4 Mon Sep 17 00:00:00 2001
From: Felix Mulder <felix.mulder@gmail.com>
Date: Wed, 27 Apr 2016 15:08:26 +0200
Subject: Revert Scanners and Tokens to their original form

Since we decided to go with the non dotty-scanner approach these are
unnecessary to have altered, might just as well revert them.
---
 src/dotty/tools/dotc/parsing/Scanners.scala | 49 +++++++----------------------
 src/dotty/tools/dotc/parsing/Tokens.scala   |  7 +----
 2 files changed, 12 insertions(+), 44 deletions(-)

(limited to 'src')

diff --git a/src/dotty/tools/dotc/parsing/Scanners.scala b/src/dotty/tools/dotc/parsing/Scanners.scala
index 91332b01c..6fe86502f 100644
--- a/src/dotty/tools/dotc/parsing/Scanners.scala
+++ b/src/dotty/tools/dotc/parsing/Scanners.scala
@@ -45,9 +45,6 @@ object Scanners {
     /** the string value of a literal */
     var strVal: String = null
 
-    /** the started parsing of a literal */
-    var startedLiteral: String = null
-
     /** the base of a number */
     var base: Int = 0
 
@@ -177,13 +174,8 @@ object Scanners {
 
   }
 
-  class Scanner(
-    source: SourceFile,
-    override val startFrom: Offset = 0,
-    preserveWhitespace: Boolean = false
-  )(implicit ctx: Context) extends ScannerCommon(source)(ctx) {
-    val keepComments       = ctx.settings.YkeepComments.value
-    val whitespace         = new StringBuilder
+  class Scanner(source: SourceFile, override val startFrom: Offset = 0)(implicit ctx: Context) extends ScannerCommon(source)(ctx) {
+    val keepComments = ctx.settings.YkeepComments.value
 
     /** All doc comments as encountered, each list contains doc comments from
      *  the same block level. Starting with the deepest level and going upward
@@ -247,13 +239,13 @@ object Scanners {
 
     /** Are we directly in a string interpolation expression?
      */
-    def inStringInterpolation =
+    private def inStringInterpolation =
       sepRegions.nonEmpty && sepRegions.head == STRINGLIT
 
     /** Are we directly in a multiline string interpolation expression?
      *  @pre inStringInterpolation
      */
-    def inMultiLineInterpolation =
+    private def inMultiLineInterpolation =
       inStringInterpolation && sepRegions.tail.nonEmpty && sepRegions.tail.head == STRINGPART
 
     /** read next token and return last offset
@@ -324,7 +316,7 @@ object Scanners {
         token = if (pastBlankLine()) NEWLINES else NEWLINE
       }
 
-      if (!preserveWhitespace) postProcessToken()
+      postProcessToken()
       // print("[" + this +"]")
     }
 
@@ -383,20 +375,9 @@ object Scanners {
       offset = charOffset - 1
       (ch: @switch) match {
         case ' ' | '\t' | CR | LF | FF =>
-          if (preserveWhitespace) {
-            while ((' ' :: '\t' :: CR :: LF :: FF :: Nil) contains ch) {
-              whitespace += ch
-              nextChar()
-            }
-            token  = WHITESPACE
-            strVal = whitespace.toString
-            whitespace.clear()
-          } else {
-            nextChar()
-            fetchToken()
-          }
-        case c @ (
-             'A' | 'B' | 'C' | 'D' | 'E' |
+          nextChar()
+          fetchToken()
+        case 'A' | 'B' | 'C' | 'D' | 'E' |
              'F' | 'G' | 'H' | 'I' | 'J' |
              'K' | 'L' | 'M' | 'N' | 'O' |
              'P' | 'Q' | 'R' | 'S' | 'T' |
@@ -407,14 +388,12 @@ object Scanners {
              'k' | 'l' | 'm' | 'n' | 'o' |
              'p' | 'q' | 'r' | 's' | 't' |
              'u' | 'v' | 'w' | 'x' | 'y' |
-             'z') =>
+             'z' =>
           putChar(ch)
           nextChar()
           getIdentRest()
-          if (ch == '"' && token == IDENTIFIER) {
+          if (ch == '"' && token == IDENTIFIER)
             token = INTERPOLATIONID
-            startedLiteral = "\""
-          }
         case '<' => // is XMLSTART?
           def fetchLT() = {
             val last = if (charOffset >= 2) buf(charOffset - 2) else ' '
@@ -515,11 +494,9 @@ object Scanners {
               getLitChar()
               if (ch == '\'') {
                 nextChar()
-                startedLiteral = null
                 token = CHARLIT
                 setStrVal()
               } else {
-                startedLiteral = "\'"
                 error("unclosed character literal")
               }
             }
@@ -709,12 +686,8 @@ object Scanners {
       if (ch == '"') {
         setStrVal()
         nextChar()
-        startedLiteral = null
         token = STRINGLIT
-      } else {
-        startedLiteral = "\""
-        error("unclosed string literal")
-      }
+      } else error("unclosed string literal")
     }
 
     private def getRawStringLit(): Unit = {
diff --git a/src/dotty/tools/dotc/parsing/Tokens.scala b/src/dotty/tools/dotc/parsing/Tokens.scala
index 3ca86d624..b490cd133 100644
--- a/src/dotty/tools/dotc/parsing/Tokens.scala
+++ b/src/dotty/tools/dotc/parsing/Tokens.scala
@@ -141,7 +141,7 @@ abstract class TokensCommon {
 
 object Tokens extends TokensCommon {
   final val minToken = EMPTY
-  final val maxToken = COMMENT
+  final val maxToken = XMLSTART
 
   final val INTERPOLATIONID = 10;  enter(INTERPOLATIONID, "string interpolator")
   final val SYMBOLLIT = 11;        enter(SYMBOLLIT, "symbol literal") // TODO: deprecate
@@ -188,11 +188,6 @@ object Tokens extends TokensCommon {
   /** XML mode */
   final val XMLSTART = 96;         enter(XMLSTART, "$XMLSTART$<") // TODO: deprecate
 
-  /** Whitespace */
-  final val WHITESPACE = 97;       enter(WHITESPACE, "whitespace")
-  final val COMMENT = 98;          enter(COMMENT, "comment")
-
-
   final val alphaKeywords = tokenRange(IF, FORSOME)
   final val symbolicKeywords = tokenRange(USCORE, VIEWBOUND)
   final val symbolicTokens = tokenRange(COMMA, VIEWBOUND)
-- 
cgit v1.2.3