path: root/src/interactive/scala/tools/nsc/interactive/Lexer.scala

package scala.tools.nsc.interactive

import java.io.Reader

/** Companion object of class `Lexer` which defines tokens and some utility concepts
 *  used for tokens and lexers
 */
object Lexer {

  /** An exception raised if an input does not correspond to what's expected
   *  @param   rdr   the lexer from which the bad input is read
   *  @param   msg   the error message
   */
  class MalformedInput(val rdr: Lexer, val msg: String) extends Exception("Malformed JSON input at "+rdr.tokenPos+": "+msg)

  /** The class of tokens, i.e. descriptions of input words (or: lexemes).
   *  @param str    the characters making up this token
   */
  class Token(val str: String) {
    override def toString = str
  }

  /** A subclass of `Token` representing single-character delimiters
   *  @param char the delimiter character making up this token
   */
  case class Delim(char: Char) extends Token(s"'$char'")

  /** A subclass of token representing integer literals */
  case class IntLit(override val str: String) extends Token(str)

  /** A subclass of token representing floating point literals */
  case class FloatLit(override val str: String) extends Token(str)

  /** A subclass of token representing string literals */
  case class StringLit(override val str: String) extends Token(str) {
    override def toString = quoted(str)
  }

  /** The `true` token */
  val TrueLit = new Token("true")

  /** The `false` token */
  val FalseLit = new Token("false")

  /** The `null` token */
  val NullLit = new Token("null")

  /** The '`(`' token */
  val LParen = new Delim('(')

  /** The '`)`' token */
  val RParen = new Delim(')')

  /** The '`{`' token */
  val LBrace = new Delim('{')

  /** The '`}`' token */
  val RBrace = new Delim('}')

  /** The '`[`' token */
  val LBracket = new Delim('[')

  /** The '`]`' token */
  val RBracket = new Delim(']')

  /** The '`,`' token */
  val Comma = new Delim(',')

  /** The '`:`' token */
  val Colon = new Delim(':')

  /** The token representing end of input */
  val EOF = new Token("<end of input>")

  private def toUDigit(ch: Int): Char = {
    val d = ch & 0xF
    (if (d < 10) d + '0' else d - 10 + 'A').toChar
  }

  private def addToStr(buf: StringBuilder, ch: Char) {
    ch match {
      case '"' => buf ++= "\\\""
      case '\b' => buf ++= "\\b"
      case '\f' => buf ++= "\\f"
      case '\n' => buf ++= "\\n"
      case '\r' => buf ++= "\\r"
      case '\t' => buf ++= "\\t"
      case '\\' => buf ++= "\\\\"
      case _ =>
        if (' ' <= ch && ch < 128) buf += ch
        else buf ++= "\\u" += toUDigit(ch >>> 12) += toUDigit(ch >>> 8) += toUDigit(ch >>> 4) += toUDigit(ch.toInt)
    }
  }

  /** Returns given string enclosed in `"`-quotes with all string characters escaped
   *  so that they correspond to the JSON standard.
   *  Characters that escaped are:  `"`, `\b`, `\f`, `\n`, `\r`, `\t`, `\`.
   *  Furthermore, every other character which is not in the ASCII range 32-127 is
   *  escaped as a four hex-digit unicode character of the form `\ u x x x x`.
   *  @param   str   the string to be quoted
   */
  def quoted(str: String): String = {
    val buf = new StringBuilder += '\"'
    str foreach (addToStr(buf, _))
    buf += '\"'
    buf.toString
  }

  private val BUF_SIZE = 2 << 16
}

import Lexer._

/** A simple lexer for tokens as they are used in JSON, plus parens `(`, `)`
 *  Tokens understood are:
 *
 *  `(`, `)`, `[`, `]`, `{`, `}`, `:`, `,`, `true`, `false`, `null`,
 *  strings (syntax as in JSON),
 *  integer numbers (syntax as in JSON: -?(0|\d+)
 *  floating point numbers (syntax as in JSON: -?(0|\d+)(\.\d+)?((e|E)(+|-)?\d+)?)
 *  The end of input is represented as its own token, EOF.
 *  Lexers can keep one token lookahead
 *
 * @param rd   the reader from which characters are read.
 */
class Lexer(rd: Reader) {

  /** The last-read character */
  var ch: Char = 0

  /** The number of characters read so far */
  var pos: Long = 0

  /** The last-read token */
  var token: Token = _

  /** The number of characters read before the start of the last-read token */
  var tokenPos: Long = 0

  private var atEOF: Boolean = false
  private val buf = new Array[Char](BUF_SIZE)
  private var nread: Int = 0
  private var bp = 0

  /** Reads next character into `ch` */
  def nextChar() {
    assert(!atEOF)
    if (bp == nread) {
      nread = rd.read(buf)
      bp = 0
      if (nread <= 0) { ch = 0; atEOF = true; return }
    }
    ch = buf(bp)
    bp += 1
    pos += 1
  }

  /** If last-read character equals given character, reads next character,
   *  otherwise raises an error
   *  @param  c   the given character to compare with last-read character
   *  @throws  MalformedInput if character does not match
   */
  def acceptChar(c: Char) = if (ch == c) nextChar() else error("'"+c+"' expected")

  private val sb = new StringBuilder

  private def putChar() {
    sb += ch; nextChar()
  }

  private def putAcceptString(str: String) {
    str foreach acceptChar
    sb ++= str
  }

  /** Skips whitespace and reads next lexeme into `token`
   *  @throws  MalformedInput if lexeme not recognized as a valid token
   */
  def nextToken() {
    sb.clear()
    while (!atEOF && ch <= ' ') nextChar()
    tokenPos = pos - 1
    if (atEOF) token = EOF
    else ch match {
      case '(' => putChar(); token = LParen
      case ')' => putChar(); token = RParen
      case '{' => putChar(); token = LBrace
      case '}' => putChar(); token = RBrace
      case '[' => putChar(); token = LBracket
      case ']' => putChar(); token = RBracket
      case ',' => putChar(); token = Comma
      case ':' => putChar(); token = Colon
      case 't' => putAcceptString("true"); token = TrueLit
      case 'f' => putAcceptString("false"); token = FalseLit
      case 'n' => putAcceptString("null"); token = NullLit
      case '"' => getString()
      case '-' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' => getNumber()
      case _ => error("unrecognized start of token: '"+ch+"'")
    }
    //println("["+token+"]")
  }

  /** Reads a string literal, and forms a `StringLit` token from it.
   *  Last-read input character `ch` must be opening `"`-quote.
   *  @throws  MalformedInput if lexeme not recognized as a string literal.
   */
  def getString() {
    def udigit() = {
      nextChar()
      if ('0' <= ch && ch <= '9') ch - '9'
      else if ('A' <= ch && ch <= 'F') ch - 'A' + 10
      else if ('a' <= ch && ch <= 'f') ch - 'a' + 10
      else error("illegal unicode escape character: '"+ch+"'")
    }
    val delim = ch
    nextChar()
    while (ch != delim && ch >= ' ') {
      if (ch == '\\') {
        nextChar()
        ch match {
          case '\'' => sb += '\''
          case '"' => sb += '"'
          case '\\' => sb += '\\'
          case '/' => sb += '/'
          case 'b' => sb += '\b'
          case 'f' => sb += '\f'
          case 'n' => sb += '\n'
          case 'r' => sb += '\r'
          case 't' => sb += '\t'
          case 'u' => sb += (udigit() << 12 | udigit() << 8 | udigit() << 4 | udigit()).toChar
          case _ => error("illegal escape character: '"+ch+"'")
        }
        nextChar()
      } else {
        putChar()
      }
    }
    acceptChar(delim)
    token = StringLit(sb.toString)
  }

  /** Reads a numeric literal, and forms an `IntLit` or `FloatLit` token from it.
   *  Last-read input character `ch` must be either `-` or a digit.
   *  @throws  MalformedInput if lexeme not recognized as a numeric literal.
   */
  def getNumber() {
    def digit() =
      if ('0' <= ch && ch <= '9') putChar()
      else error("<digit> expected")
    def digits() =
      do { digit() } while ('0' <= ch && ch <= '9')
    var isFloating = false
    if (ch == '-') putChar()
    if (ch == '0') digit()
    else digits()
    if (ch == '.') {
      isFloating = true
      putChar()
      digits()
    }
    if (ch == 'e' || ch == 'E') {
      isFloating = true
      putChar()
      if (ch == '+' || ch == '-') putChar()
      digits()
    }
    token = if (isFloating) FloatLit(sb.toString) else IntLit(sb.toString)
  }

  /** If current token equals given token, reads next token, otherwise raises an error.
   *  @param  t   the given token to compare current token with
   *  @throws MalformedInput  if the two tokens do not match.
   */
  def accept(t: Token) {
    if (token == t) nextToken()
    else error(t+" expected, but "+token+" found")
  }

  /** The current token is a delimiter consisting of given character, reads next token,
   *  otherwise raises an error.
   *  @param  ch   the given delimiter character to compare current token with
   *  @throws MalformedInput  if the current token `token` is not a delimiter, or
   *                          consists of a character different from `c`.
   */
  def accept(ch: Char) {
    token match {
      case Delim(`ch`) => nextToken()
      case _ => accept(Delim(ch))
    }
  }

  /** Always throws a `MalformedInput` exception with given error message.
   *  @param msg  the error message
   */
  def error(msg: String) = throw new MalformedInput(this, msg)

  nextChar()
  nextToken()
}