integrating combinators into trunk

2 files changed, 73 insertions, 0 deletions

diff --git a/src/library/scala/util/parsing/syntax/StdTokens.scala b/src/library/scala/util/parsing/syntax/StdTokens.scala
new file mode 100644
index 0000000000..0f7a70bb7d
--- /dev/null
+++ b/src/library/scala/util/parsing/syntax/StdTokens.scala
@@ -0,0 +1,35 @@
+/*                     __                                               *\
+**     ________ ___   / /  ___     Scala API                            **
+**    / __/ __// _ | / /  / _ |    (c) 2006-2007, LAMP/EPFL             **
+**  __\ \/ /__/ __ |/ /__/ __ |                                         **
+** /____/\___/_/ |_/____/_/ | |                                         **
+**                          |/                                          **
+\*                                                                      */
+
+package scala.util.parsing.syntax
+
+/** This component provides the standard `Token's for a simple, Scala-like language.
+ *
+ * @author Martin Odersky, Adriaan Moors
+ */
+trait StdTokens extends Tokens {
+  /** The class of keyword tokens */
+  case class Keyword(chars: String) extends Token {
+    override def toString = "`"+chars+"'"
+  }
+
+  /** The class of numeric literal tokens */
+  case class NumericLit(chars: String) extends Token {
+    override def toString = chars
+  }
+
+  /** The class of string literal tokens */
+  case class StringLit(chars: String) extends Token {
+    override def toString = "\""+chars+"\""
+  }
+
+  /** The class of identifier tokens */
+  case class Identifier(chars: String) extends Token {
+    override def toString = "identifier "+chars
+  }
+}
diff --git a/src/library/scala/util/parsing/syntax/Tokens.scala b/src/library/scala/util/parsing/syntax/Tokens.scala
new file mode 100644
index 0000000000..748518949b
--- /dev/null
+++ b/src/library/scala/util/parsing/syntax/Tokens.scala
@@ -0,0 +1,38 @@
+/*                     __                                               *\
+**     ________ ___   / /  ___     Scala API                            **
+**    / __/ __// _ | / /  / _ |    (c) 2006-2007, LAMP/EPFL             **
+**  __\ \/ /__/ __ |/ /__/ __ |                                         **
+** /____/\___/_/ |_/____/_/ | |                                         **
+**                          |/                                          **
+\*                                                                      */
+
+package scala.util.parsing.syntax
+
+/** This component provides the notion of `Token', the unit of information that is passed from lexical
+ * parsers in the `Lexical' component to the parsers in the `TokenParsers' component.
+ *
+ * @author Martin Odersky, Adriaan Moors
+ */
+trait Tokens {
+  /** Objects of this type are produced by a lexical parser or ``scanner'', and consumed by a parser
+   * {@see scala.util.parsing.combinator.syntactical.TokenParsers}.
+   */
+  abstract class Token {
+    def chars: String
+  }
+
+  /** A class of error tokens. Error tokens are used to communicate
+   *  errors detected during lexical analysis
+   */
+  case class ErrorToken(msg: String) extends Token {
+    def chars = "*** error: "+msg
+  }
+
+  /** A class for end-of-file tokens */
+  case object EOF extends Token {
+    def chars = "<eof>"
+  }
+
+  /** This token is produced by a scanner {@see Scanner} when scanning failed. */
+  def errorToken(msg: String): Token = new ErrorToken(msg)
+}