blob: 20a23a8492d3c528241d8493df07a13e69b81f0f (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
|
package scala.util.parsing;
class SimpleTokenizer(in: Iterator[char], delimiters: String) extends Iterator[String] {
val tracing = false;
private def max(x: int, y: char): int = if (x < y) y else x;
private def delimArray: Array[boolean] = {
val ds = List.fromString(delimiters);
val da = new Array[boolean]((0 /: ds)(max) + 1);
for (val ch <- ds) { da(ch) = true }
da
}
private val isdelim = delimArray;
private def isDelimiter(ch: int) = ch >= 0 && ch < isdelim.length && isdelim(ch);
private val EOI = -1;
private def nextChar: int = if (in.hasNext) in.next else EOI;
private var ch: int = nextChar;
private val buf = new StringBuffer();
def hasNext: boolean = ch != EOI;
def next: String = {
while (ch <= ' ' && ch != EOI) ch = nextChar;
if (ch == EOI) ""
else {
buf.setLength(0);
if (isDelimiter(ch)) {
buf append ch.asInstanceOf[char]; ch = nextChar
} else {
while (ch > ' ' && ch != EOI && !isDelimiter(ch)) {
buf append ch.asInstanceOf[char]; ch = nextChar;
}
}
if (tracing) System.out.println("<" + buf.toString() + ">");
buf.toString()
}
}
}
|
