aboutsummaryrefslogtreecommitdiff
path: root/src/dotty/tools/dotc/parsing/Tokens.scala
diff options
context:
space:
mode:
authorMartin Odersky <odersky@gmail.com>2013-04-17 09:48:22 +0200
committerMartin Odersky <odersky@gmail.com>2013-04-17 10:16:22 +0200
commitca8dc7ada663e44aafe470944dd17256dbde151c (patch)
treed15939e204042e358e0c83064250f1f18c1c4f25 /src/dotty/tools/dotc/parsing/Tokens.scala
parente32fedb6844eab11a27e365a570b2033a0f6f78d (diff)
downloaddotty-ca8dc7ada663e44aafe470944dd17256dbde151c.tar.gz
dotty-ca8dc7ada663e44aafe470944dd17256dbde151c.tar.bz2
dotty-ca8dc7ada663e44aafe470944dd17256dbde151c.zip
Scanners added.
Moving Positions, Chars to new packages. Added Source positions. Added untyped trees module. Factored out behavior between typed and untyped trees.
Diffstat (limited to 'src/dotty/tools/dotc/parsing/Tokens.scala')
-rw-r--r--src/dotty/tools/dotc/parsing/Tokens.scala171
1 files changed, 171 insertions, 0 deletions
diff --git a/src/dotty/tools/dotc/parsing/Tokens.scala b/src/dotty/tools/dotc/parsing/Tokens.scala
new file mode 100644
index 000000000..f573df49d
--- /dev/null
+++ b/src/dotty/tools/dotc/parsing/Tokens.scala
@@ -0,0 +1,171 @@
+package dotty.tools
+package dotc
+package parsing
+
+import collection.mutable
+import collection.immutable.BitSet
+import scala.annotation.switch
+
+object Tokens {
+
+ final val minToken = EMPTY
+ final val maxToken = XMLSTART
+
+ type TokenSet = BitSet
+
+ def tokenRange(lo: Int, hi: Int): TokenSet = BitSet(lo to hi: _*)
+
+ def showTokenDetailed(token: Int) = debugString(token)
+
+ def showToken(token: Int) = {
+ val str = tokenString(token)
+ if (keywords contains token) s"'$str'" else str
+ }
+
+ val tokenString, debugString = new Array[String](maxToken + 1)
+
+ def enter(token: Int, str: String, debugStr: String = ""): Unit = {
+ tokenString(token) = str
+ debugString(token) = if (debugStr.isEmpty) str else debugStr
+ }
+
+ /** special tokens */
+ final val EMPTY = 0; enter(EMPTY, "<empty>") // a missing token, used in lookahead
+ final val ERROR = 1; enter(ERROR, "erroneous token") // an erroneous token
+ final val EOF = 2; enter(EOF, "eof")
+
+ /** literals */
+ final val CHARLIT = 3; enter(CHARLIT, "character literal")
+ final val INTLIT = 4; enter(INTLIT, "integer literal")
+ final val LONGLIT = 5; enter(LONGLIT, "long literal")
+ final val FLOATLIT = 6; enter(FLOATLIT, "float literal")
+ final val DOUBLELIT = 7; enter(DOUBLELIT, "double literal")
+ final val STRINGLIT = 8; enter(STRINGLIT, "string literal")
+ final val STRINGPART = 9; enter(STRINGPART, "string literal", "string literal part")
+ final val INTERPOLATIONID = 10; enter(INTERPOLATIONID, "string interpolator")
+ final val SYMBOLLIT = 11; enter(SYMBOLLIT, "symbol literal") // TODO: deprecate
+
+ final val literalTokens = tokenRange(CHARLIT, SYMBOLLIT)
+
+ /** identifiers */
+ final val IDENTIFIER = 12; enter(IDENTIFIER, "identifier")
+ final val BACKQUOTED_IDENT = 13; enter(BACKQUOTED_IDENT, "identifier", "backquoted ident")
+
+ final val identifierTokens = BitSet(IDENTIFIER, BACKQUOTED_IDENT)
+
+ def isIdentifier(token : Int) =
+ token >= IDENTIFIER && token <= BACKQUOTED_IDENT
+
+ /** alphabetic keywords */
+ final val IF = 20; enter(IF, "if")
+ final val FOR = 21; enter(FOR, "for")
+ final val ELSE = 22; enter(ELSE, "else")
+ final val THIS = 23; enter(THIS, "this")
+ final val NULL = 24; enter(NULL, "null")
+ final val NEW = 25; enter(NEW, "new")
+ final val WITH = 26; enter(WITH, "with")
+ final val SUPER = 27; enter(SUPER, "super")
+ final val CASE = 28; enter(CASE, "case")
+ final val CASECLASS = 29; enter(CASECLASS, "case class")
+ final val CASEOBJECT = 30; enter(CASEOBJECT, "case object")
+ final val VAL = 31; enter(VAL, "val")
+ final val ABSTRACT = 32; enter(ABSTRACT, "abstract")
+ final val FINAL = 33; enter(FINAL, "final")
+ final val PRIVATE = 34; enter(PRIVATE, "private")
+ final val PROTECTED = 35; enter(PROTECTED, "protected")
+ final val OVERRIDE = 36; enter(OVERRIDE, "override")
+ final val IMPLICIT = 37; enter(IMPLICIT, "implicit")
+ final val VAR = 38; enter(VAR, "var")
+ final val DEF = 39; enter(DEF, "def")
+ final val TYPE = 40; enter(TYPE, "type")
+ final val EXTENDS = 41; enter(EXTENDS, "extends")
+ final val TRUE = 42; enter(TRUE, "true")
+ final val FALSE = 43; enter(FALSE, "false")
+ final val OBJECT = 44; enter(OBJECT, "object")
+ final val CLASS = 45; enter(CLASS, "class")
+ final val IMPORT = 46; enter(IMPORT, "import")
+ final val PACKAGE = 47; enter(PACKAGE, "package")
+ final val YIELD = 48; enter(YIELD, "yield")
+ final val DO = 49; enter(DO, "do")
+ final val TRAIT = 50; enter(TRAIT, "trait")
+ final val SEALED = 51; enter(SEALED, "sealed")
+ final val THROW = 52; enter(THROW, "throw")
+ final val TRY = 53; enter(TRY, "try")
+ final val CATCH = 54; enter(CATCH, "catch")
+ final val FINALLY = 55; enter(FINALLY, "finally")
+ final val WHILE = 56; enter(WHILE, "while")
+ final val RETURN = 57; enter(RETURN, "return")
+ final val MATCH = 58; enter(MATCH, "match")
+ final val FORSOME = 59; enter(FORSOME, "forSome") // TODO: deprecate
+ final val LAZY = 61; enter(LAZY, "lazy")
+ final val THEN = 62; enter(THEN, "then")
+
+ final val alphaKeywords = tokenRange(IF, LAZY)
+
+ /** special symbols */
+ final val COMMA = 70; enter(COMMA, "','")
+ final val SEMI = 71; enter(DOT, "'.'")
+ final val DOT = 72; enter(SEMI, "';'")
+ final val NEWLINE = 78; enter(NEWLINE, "';'", "new line")
+ final val NEWLINES = 79; enter(NEWLINES, "';'", "new lines")
+
+ /** special keywords */
+ final val USCORE = 73; enter(USCORE, "_")
+ final val COLON = 74; enter(COLON, ":")
+ final val EQUALS = 75; enter(EQUALS, "==")
+ final val LARROW = 76; enter(LARROW, "<-")
+ final val ARROW = 77; enter(ARROW, "=>")
+ final val SUBTYPE = 80; enter(SUBTYPE, "<:")
+ final val SUPERTYPE = 81; enter(SUPERTYPE, ">:")
+ final val HASH = 82; enter(HASH, "#")
+ final val AT = 83; enter(AT, "@")
+ final val VIEWBOUND = 84; enter(VIEWBOUND, "<%") // TODO: deprecate
+
+ final val symbolicKeywords = tokenRange(USCORE, VIEWBOUND)
+ final val symbolicTokens = tokenRange(COMMA, VIEWBOUND)
+ final val keywords = alphaKeywords | symbolicKeywords
+
+ /** parentheses */
+ final val LPAREN = 90; enter(LPAREN, "'('")
+ final val RPAREN = 91; enter(RPAREN, "')'")
+ final val LBRACKET = 92; enter(LBRACKET, "'['")
+ final val RBRACKET = 93; enter(RBRACKET, "']'")
+ final val LBRACE = 94; enter(LBRACE, "'{'")
+ final val RBRACE = 95; enter(RBRACE, "'}'")
+
+ /** XML mode */
+ final val XMLSTART = 96; enter(XMLSTART, "$XMLSTART$<") // TODO: deprecate
+
+ final val allTokens = tokenRange(minToken, maxToken)
+
+ final val atomicExprTokens = literalTokens | identifierTokens | BitSet(
+ USCORE, NULL, THIS, SUPER, TRUE, FALSE, RETURN, XMLSTART)
+
+ final val canStartExpressionTokens = atomicExprTokens | BitSet(
+ LBRACE, LPAREN, IF, DO, WHILE, FOR, NEW, TRY, THROW)
+
+ final val canStartTypeTokens = literalTokens | identifierTokens | BitSet(
+ THIS, SUPER, USCORE, LPAREN, AT)
+
+ final val templateIntroTokens = BitSet(CLASS, TRAIT, OBJECT, CASECLASS, CASEOBJECT)
+
+ final val dclIntroTokens = BitSet(DEF, VAL, VAR, TYPE)
+
+ final val defIntroTokens = templateIntroTokens | dclIntroTokens
+
+ final val localModifierTokens = BitSet(
+ ABSTRACT, FINAL, SEALED, IMPLICIT, LAZY)
+
+ final val modifierTokens = localModifierTokens | BitSet(
+ PRIVATE, PROTECTED, OVERRIDE)
+
+ /** Is token only legal as start of statement (eof also included)? */
+ final val mustStartStatTokens = defIntroTokens | modifierTokens | BitSet(
+ CASE, IMPORT, PACKAGE)
+
+ final val canStartStatTokens = canStartExpressionTokens | mustStartStatTokens | BitSet(
+ AT)
+
+ final val canEndStatTokens = atomicExprTokens | BitSet(
+ TYPE, RPAREN, RBRACE, RBRACKET)
+}