diff options
Diffstat (limited to 'src/dotty/tools/dotc/parsing/Tokens.scala')
-rw-r--r-- | src/dotty/tools/dotc/parsing/Tokens.scala | 143 |
1 files changed, 100 insertions, 43 deletions
diff --git a/src/dotty/tools/dotc/parsing/Tokens.scala b/src/dotty/tools/dotc/parsing/Tokens.scala index 09124d0d1..226a3710d 100644 --- a/src/dotty/tools/dotc/parsing/Tokens.scala +++ b/src/dotty/tools/dotc/parsing/Tokens.scala @@ -3,11 +3,10 @@ package dotc package parsing import collection.immutable.BitSet +import core.Decorators._ -object Tokens { - - final val minToken = EMPTY - final val maxToken = XMLSTART +abstract class TokensCommon { + val maxToken: Int type Token = Int type TokenSet = BitSet @@ -24,6 +23,7 @@ object Tokens { val tokenString, debugString = new Array[String](maxToken + 1) def enter(token: Int, str: String, debugStr: String = ""): Unit = { + assert(tokenString(token) == null) tokenString(token) = str debugString(token) = if (debugStr.isEmpty) str else debugStr } @@ -41,17 +41,12 @@ object Tokens { final val DOUBLELIT = 7; enter(DOUBLELIT, "double literal") final val STRINGLIT = 8; enter(STRINGLIT, "string literal") final val STRINGPART = 9; enter(STRINGPART, "string literal", "string literal part") - final val INTERPOLATIONID = 10; enter(INTERPOLATIONID, "string interpolator") - final val SYMBOLLIT = 11; enter(SYMBOLLIT, "symbol literal") // TODO: deprecate + //final val INTERPOLATIONID = 10; enter(INTERPOLATIONID, "string interpolator") + //final val SYMBOLLIT = 11; enter(SYMBOLLIT, "symbol literal") // TODO: deprecate /** identifiers */ final val IDENTIFIER = 12; enter(IDENTIFIER, "identifier") - final val BACKQUOTED_IDENT = 13; enter(BACKQUOTED_IDENT, "identifier", "backquoted ident") - - final val identifierTokens = BitSet(IDENTIFIER, BACKQUOTED_IDENT) - - def isIdentifier(token : Int) = - token >= IDENTIFIER && token <= BACKQUOTED_IDENT + //final val BACKQUOTED_IDENT = 13; enter(BACKQUOTED_IDENT, "identifier", "backquoted ident") /** alphabetic keywords */ final val IF = 20; enter(IF, "if") @@ -60,67 +55,63 @@ object Tokens { final val THIS = 23; enter(THIS, "this") final val NULL = 24; enter(NULL, "null") final val NEW = 25; enter(NEW, "new") - final val WITH = 26; enter(WITH, "with") + //final val WITH = 26; enter(WITH, "with") final val SUPER = 27; enter(SUPER, "super") - final val CASE = 28; enter(CASE, "case") - final val CASECLASS = 29; enter(CASECLASS, "case class") - final val CASEOBJECT = 30; enter(CASEOBJECT, "case object") - final val VAL = 31; enter(VAL, "val") + //final val CASE = 28; enter(CASE, "case") + //final val CASECLASS = 29; enter(CASECLASS, "case class") + //final val CASEOBJECT = 30; enter(CASEOBJECT, "case object") + //final val VAL = 31; enter(VAL, "val") final val ABSTRACT = 32; enter(ABSTRACT, "abstract") final val FINAL = 33; enter(FINAL, "final") final val PRIVATE = 34; enter(PRIVATE, "private") final val PROTECTED = 35; enter(PROTECTED, "protected") final val OVERRIDE = 36; enter(OVERRIDE, "override") - final val IMPLICIT = 37; enter(IMPLICIT, "implicit") - final val VAR = 38; enter(VAR, "var") - final val DEF = 39; enter(DEF, "def") - final val TYPE = 40; enter(TYPE, "type") + //final val IMPLICIT = 37; enter(IMPLICIT, "implicit") + //final val VAR = 38; enter(VAR, "var") + //final val DEF = 39; enter(DEF, "def") + //final val TYPE = 40; enter(TYPE, "type") final val EXTENDS = 41; enter(EXTENDS, "extends") final val TRUE = 42; enter(TRUE, "true") final val FALSE = 43; enter(FALSE, "false") - final val OBJECT = 44; enter(OBJECT, "object") + //final val OBJECT = 44; enter(OBJECT, "object") final val CLASS = 45; enter(CLASS, "class") final val IMPORT = 46; enter(IMPORT, "import") final val PACKAGE = 47; enter(PACKAGE, "package") - final val YIELD = 48; enter(YIELD, "yield") + //final val YIELD = 48; enter(YIELD, "yield") final val DO = 49; enter(DO, "do") - final val TRAIT = 50; enter(TRAIT, "trait") - final val SEALED = 51; enter(SEALED, "sealed") + //final val TRAIT = 50; enter(TRAIT, "trait") + //final val SEALED = 51; enter(SEALED, "sealed") final val THROW = 52; enter(THROW, "throw") final val TRY = 53; enter(TRY, "try") final val CATCH = 54; enter(CATCH, "catch") final val FINALLY = 55; enter(FINALLY, "finally") final val WHILE = 56; enter(WHILE, "while") final val RETURN = 57; enter(RETURN, "return") - final val MATCH = 58; enter(MATCH, "match") - final val LAZY = 59; enter(LAZY, "lazy") - final val THEN = 60; enter(THEN, "then") - final val FORSOME = 61; enter(FORSOME, "forSome") // TODO: deprecate - - final val alphaKeywords = tokenRange(IF, FORSOME) + //final val MATCH = 58; enter(MATCH, "match") + //final val LAZY = 59; enter(LAZY, "lazy") + //final val THEN = 60; enter(THEN, "then") + //final val FORSOME = 61; enter(FORSOME, "forSome") // TODO: deprecate /** special symbols */ final val COMMA = 70; enter(COMMA, "','") final val SEMI = 71; enter(DOT, "'.'") final val DOT = 72; enter(SEMI, "';'") - final val NEWLINE = 78; enter(NEWLINE, "end of statement", "new line") - final val NEWLINES = 79; enter(NEWLINES, "end of statement", "new lines") + //final val NEWLINE = 78; enter(NEWLINE, "end of statement", "new line") + //final val NEWLINES = 79; enter(NEWLINES, "end of statement", "new lines") /** special keywords */ - final val USCORE = 73; enter(USCORE, "_") + //final val USCORE = 73; enter(USCORE, "_") final val COLON = 74; enter(COLON, ":") final val EQUALS = 75; enter(EQUALS, "=") - final val LARROW = 76; enter(LARROW, "<-") - final val ARROW = 77; enter(ARROW, "=>") - final val SUBTYPE = 80; enter(SUBTYPE, "<:") - final val SUPERTYPE = 81; enter(SUPERTYPE, ">:") - final val HASH = 82; enter(HASH, "#") + //final val LARROW = 76; enter(LARROW, "<-") + //final val ARROW = 77; enter(ARROW, "=>") + //final val SUBTYPE = 80; enter(SUBTYPE, "<:") + //final val SUPERTYPE = 81; enter(SUPERTYPE, ">:") + //final val HASH = 82; enter(HASH, "#") final val AT = 83; enter(AT, "@") - final val VIEWBOUND = 84; enter(VIEWBOUND, "<%") // TODO: deprecate + //final val VIEWBOUND = 84; enter(VIEWBOUND, "<%") // TODO: deprecate - final val symbolicKeywords = tokenRange(USCORE, VIEWBOUND) - final val symbolicTokens = tokenRange(COMMA, VIEWBOUND) - final val keywords = alphaKeywords | symbolicKeywords + val keywords: TokenSet /** parentheses */ final val LPAREN = 90; enter(LPAREN, "'('") @@ -133,9 +124,75 @@ object Tokens { final val firstParen = LPAREN final val lastParen = RBRACE + def buildKeywordArray(keywords: TokenSet) = { + def start(tok: Token) = tokenString(tok).toTermName.start + def sourceKeywords = keywords.toList.filter { (kw: Token) => + val ts = tokenString(kw) + (ts != null) && !ts.contains(' ') + } + + val lastKeywordStart = sourceKeywords.map(start).max + + val arr = Array.fill(lastKeywordStart + 1)(IDENTIFIER) + for (kw <- sourceKeywords) arr(start(kw)) = kw + (lastKeywordStart, arr) + } +} + +object Tokens extends TokensCommon { + final val minToken = EMPTY + final val maxToken = XMLSTART + + final val INTERPOLATIONID = 10; enter(INTERPOLATIONID, "string interpolator") + final val SYMBOLLIT = 11; enter(SYMBOLLIT, "symbol literal") // TODO: deprecate + + final val BACKQUOTED_IDENT = 13; enter(BACKQUOTED_IDENT, "identifier", "backquoted ident") + + final val identifierTokens = BitSet(IDENTIFIER, BACKQUOTED_IDENT) + + def isIdentifier(token : Int) = + token >= IDENTIFIER && token <= BACKQUOTED_IDENT + + /** alphabetic keywords */ + final val WITH = 26; enter(WITH, "with") + final val CASE = 28; enter(CASE, "case") + final val CASECLASS = 29; enter(CASECLASS, "case class") + final val CASEOBJECT = 30; enter(CASEOBJECT, "case object") + final val VAL = 31; enter(VAL, "val") + final val IMPLICIT = 37; enter(IMPLICIT, "implicit") + final val VAR = 38; enter(VAR, "var") + final val DEF = 39; enter(DEF, "def") + final val TYPE = 40; enter(TYPE, "type") + final val OBJECT = 44; enter(OBJECT, "object") + final val YIELD = 48; enter(YIELD, "yield") + final val TRAIT = 50; enter(TRAIT, "trait") + final val SEALED = 51; enter(SEALED, "sealed") + final val MATCH = 58; enter(MATCH, "match") + final val LAZY = 59; enter(LAZY, "lazy") + final val THEN = 60; enter(THEN, "then") + final val FORSOME = 61; enter(FORSOME, "forSome") // TODO: deprecate + + /** special symbols */ + final val NEWLINE = 78; enter(NEWLINE, "end of statement", "new line") + final val NEWLINES = 79; enter(NEWLINES, "end of statement", "new lines") + + /** special keywords */ + final val USCORE = 73; enter(USCORE, "_") + final val LARROW = 76; enter(LARROW, "<-") + final val ARROW = 77; enter(ARROW, "=>") + final val SUBTYPE = 80; enter(SUBTYPE, "<:") + final val SUPERTYPE = 81; enter(SUPERTYPE, ">:") + final val HASH = 82; enter(HASH, "#") + final val VIEWBOUND = 84; enter(VIEWBOUND, "<%") // TODO: deprecate + /** XML mode */ final val XMLSTART = 96; enter(XMLSTART, "$XMLSTART$<") // TODO: deprecate + final val alphaKeywords = tokenRange(IF, FORSOME) + final val symbolicKeywords = tokenRange(USCORE, VIEWBOUND) + final val symbolicTokens = tokenRange(COMMA, VIEWBOUND) + final val keywords = alphaKeywords | symbolicKeywords + final val allTokens = tokenRange(minToken, maxToken) final val literalTokens = tokenRange(CHARLIT, SYMBOLLIT) | BitSet(TRUE, FALSE, NULL) |