*** empty log message ***

1 files changed, 56 insertions, 8 deletions

diff --git a/sources/examples/Parsers.scala b/sources/examples/Parsers.scala
index d99c3628ec..5697868d76 100644
--- a/sources/examples/Parsers.scala
+++ b/sources/examples/Parsers.scala
@@ -2,37 +2,37 @@ package examples;
 
 abstract class Parsers {
 
-  type intype;
+  type inputType;
 
   trait Parser[a] {
 
-    type Result = Option[Pair[a, intype]];
+    type Result = Option[Pair[a, inputType]];
 
-    def apply(in: intype): Result;
+    def apply(in: inputType): Result;
 
     def filter(pred: a => boolean) = new Parser[a] {
-      def apply(in: intype): Result = Parser.this.apply(in) match {
+      def apply(in: inputType): Result = Parser.this.apply(in) match {
         case None => None
         case Some(Pair(x, in1)) => if (pred(x)) Some(Pair(x, in1)) else None
       }
     }
 
     def map[b](f: a => b) = new Parser[b] {
-      def apply(in: intype): Result = Parser.this.apply(in) match {
+      def apply(in: inputType): Result = Parser.this.apply(in) match {
         case None => None
         case Some(Pair(x, in1)) => Some(Pair(f(x), in1))
       }
     }
 
     def flatMap[b](f: a => Parser[b]) = new Parser[b] {
-      def apply(in: intype): Result = Parser.this.apply(in) match {
+      def apply(in: inputType): Result = Parser.this.apply(in) match {
         case None => None
         case Some(Pair(x, in1)) => f(x).apply(in1)
       }
     }
 
     def ||| (def p: Parser[a]) = new Parser[a] {
-      def apply(in: intype): Result = Parser.this.apply(in) match {
+      def apply(in: inputType): Result = Parser.this.apply(in) match {
 	case None => p(in)
 	case s => s
       }
@@ -43,7 +43,7 @@ abstract class Parsers {
   }
 
   def succeed[a](x: a) = new Parser[a] {
-    def apply(in: intype): Result = Some(Pair(x, in))
+    def apply(in: inputType): Result = Some(Pair(x, in))
   }
 
   def rep[a](p: Parser[a]): Parser[List[a]] =
@@ -56,6 +56,53 @@ abstract class Parsers {
     (for (val x <- p) yield List(x)) ||| succeed(List());
 }
 
+class Tokenizer(in: Iterator[char], delimiters: String) extends Iterator[String] {
+
+  val EOI = 0;
+
+  def nextChar() =
+    if (in.hasNext()) in.next() else EOI;
+
+  private var ch = nextChar();
+
+  def isDelimiter(ch: Char) = {
+    var i = 0;
+    while (i < delimiters.length() && delimiters.charAt(i) != ch) { i = i + 1 }
+    i < delimiters.length()
+  }
+
+  def hasNext: boolean = ch != EOI;
+
+  private val buf = new StringBuffer;
+
+  def next: String =
+    while (ch <= ' ' && ch != EOI) nextChar();
+    if (ch == EOI) ""
+    else {
+      if (isDelimiter(ch)) ch.toString()
+      else {
+        buf.setLength(0); buf append ch;
+	while (ch > ' ' && ch != EOI && !isDelimiter(ch)) {
+          buf append ch; nextChar();
+        }
+        buf.toString()
+      }
+    }
+}
+
+abstract class TokenParsers extends Parsers {
+  type inputType = Stream[String];
+  def nextToken() = new Parser[String] {
+    def apply(in: inputType): Result =
+      if (in.isEmpty) None else Some(Pair(in.head, in.tail));
+  }
+}
+
+
+
+
+
+
 abstract class CharParsers extends Parsers {
   def any: Parser[char];
   def chr(ch: char) =
@@ -63,3 +110,4 @@ abstract class CharParsers extends Parsers {
   def chr(p: char => boolean) =
     for (val c <- any; p(c)) yield c;
 }
+abstract class