1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
|
/* __ *\
** ________ ___ / / ___ Scala API **
** / __/ __// _ | / / / _ | (c) 2006-2011, LAMP/EPFL **
** __\ \/ /__/ __ |/ /__/ __ | http://scala-lang.org/ **
** /____/\___/_/ |_/____/_/ | | **
** |/ **
\* */
package scala.util.parsing
package combinator
package lexical
import token._
import input._
/** This component provides core functionality for lexical parsers.
*
* See its subclasses [[scala.util.parsing.combinator.lexical.Lexical]] and -- most interestingly
* [[scala.util.parsing.combinator.lexical.StdLexical]], for more functionality.
*
* @author Martin Odersky, Adriaan Moors
*/
trait Scanners extends Parsers {
type Elem = Char
type Token
/** This token is produced by a scanner `Scanner` when scanning failed. */
def errorToken(msg: String): Token
/** A parser that produces a token (from a stream of characters). */
def token: Parser[Token]
/** A parser for white-space -- its result will be discarded. */
def whitespace: Parser[Any]
/** `Scanner` is essentially¹ a parser that produces `Token`s
* from a stream of characters. The tokens it produces are typically
* passed to parsers in `TokenParsers`.
*
* @note ¹ `Scanner` is really a `Reader` of `Token`s
*/
class Scanner(in: Reader[Char]) extends Reader[Token] {
/** Convenience constructor (makes a character reader out of the given string) */
def this(in: String) = this(new CharArrayReader(in.toCharArray()))
private val (tok, rest1, rest2) = whitespace(in) match {
case Success(_, in1) =>
token(in1) match {
case Success(tok, in2) => (tok, in1, in2)
case ns: NoSuccess => (errorToken(ns.msg), ns.next, skip(ns.next))
}
case ns: NoSuccess => (errorToken(ns.msg), ns.next, skip(ns.next))
}
private def skip(in: Reader[Char]) = if (in.atEnd) in else in.rest
override def source: java.lang.CharSequence = in.source
override def offset: Int = in.offset
def first = tok
def rest = new Scanner(rest2)
def pos = rest1.pos
def atEnd = in.atEnd || (whitespace(in) match { case Success(_, in1) => in1.atEnd case _ => false })
}
}
|
