1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
|
/* __ *\
** ________ ___ / / ___ Scala API **
** / __/ __// _ | / / / _ | (c) 2006-2010, LAMP/EPFL **
** __\ \/ /__/ __ |/ /__/ __ | http://scala-lang.org/ **
** /____/\___/_/ |_/____/_/ | | **
** |/ **
\* */
package scala.util.parsing
package combinator
package lexical
import token._
import input._
/** <p>
* This component provides core functionality for lexical parsers.
* </p>
* <p>
* See its subclasses {@see Lexical} and -- most interestingly
* {@see StdLexical}, for more functionality.
* </p>
*
* @author Martin Odersky, Adriaan Moors
*/
trait Scanners extends Parsers {
type Elem = Char
type Token
/** This token is produced by a scanner {@see Scanner} when scanning failed. */
def errorToken(msg: String): Token
/** a parser that produces a token (from a stream of characters) */
def token: Parser[Token]
/** a parser for white-space -- its result will be discarded */
def whitespace: Parser[Any]
/** <p>
* <code>Scanner</code> is essentially(*) a parser that produces `Token's
* from a stream of characters. The tokens it produces are typically
* passed to parsers in <code>TokenParsers</code>.
* </p>
* <p>
* Note: (*) <code>Scanner</code> is really a `Reader' of `Token's
* </p>
*/
class Scanner(in: Reader[Char]) extends Reader[Token] {
/** Convenience constructor (makes a character reader out of the given string) */
def this(in: String) = this(new CharArrayReader(in.toCharArray()))
private val (tok, rest1, rest2) = whitespace(in) match {
case Success(_, in1) =>
token(in1) match {
case Success(tok, in2) => (tok, in1, in2)
case ns: NoSuccess => (errorToken(ns.msg), ns.next, skip(ns.next))
}
case ns: NoSuccess => (errorToken(ns.msg), ns.next, skip(ns.next))
}
private def skip(in: Reader[Char]) = if (in.atEnd) in else in.rest
override def source: java.lang.CharSequence = in.source
override def offset: Int = in.offset
def first = tok
def rest = new Scanner(rest2)
def pos = rest1.pos
def atEnd = in.atEnd || (whitespace(in) match { case Success(_, in1) => in1.atEnd case _ => false })
}
}
|