Scanners added.

Moving Positions, Chars to new packages.
Added Source positions.
Added untyped trees module.
Factored out behavior between typed and untyped trees.

4 files changed, 331 insertions, 0 deletions

diff --git a/src/dotty/tools/dotc/util/Chars.scala b/src/dotty/tools/dotc/util/Chars.scala
new file mode 100644
index 000000000..99aad620a
--- /dev/null
+++ b/src/dotty/tools/dotc/util/Chars.scala
@@ -0,0 +1,96 @@
+/* NSC -- new Scala compiler
+ * Copyright 2006-2012 LAMP/EPFL
+ * @author  Martin Odersky
+ */
+package dotty.tools.dotc
+package util
+
+import scala.annotation.switch
+import java.lang.{ Character => JCharacter }
+import java.lang.{Character => JCharacter}
+import java.lang.Character.LETTER_NUMBER
+import java.lang.Character.LOWERCASE_LETTER
+import java.lang.Character.OTHER_LETTER
+import java.lang.Character.TITLECASE_LETTER
+import java.lang.Character.UPPERCASE_LETTER
+
+/** Contains constants and classifier methods for characters */
+object Chars {
+
+  final val LF = '\u000A'
+  final val FF = '\u000C'
+  final val CR = '\u000D'
+  final val SU = '\u001A'
+
+  /** Convert a character digit to an Int according to given base,
+   *  -1 if no success
+   */
+  def digit2int(ch: Char, base: Int): Int = {
+    val num = (
+      if (ch <= '9') ch - '0'
+      else if ('a' <= ch && ch <= 'z') ch - 'a' + 10
+      else if ('A' <= ch && ch <= 'Z') ch - 'A' + 10
+      else -1
+    )
+    if (0 <= num && num < base) num else -1
+  }
+  /** Buffer for creating '\ u XXXX' strings. */
+  private[this] val char2uescapeArray = Array[Char]('\\', 'u', 0, 0, 0, 0)
+
+  /** Convert a character to a backslash-u escape */
+  def char2uescape(c: Char): String = {
+    @inline def hexChar(ch: Int): Char =
+      ( if (ch < 10) '0' else 'A' - 10 ) + ch toChar
+
+    char2uescapeArray(2) = hexChar((c >> 12)     )
+    char2uescapeArray(3) = hexChar((c >>  8) % 16)
+    char2uescapeArray(4) = hexChar((c >>  4) % 16)
+    char2uescapeArray(5) = hexChar((c      ) % 16)
+
+    new String(char2uescapeArray)
+  }
+
+  /** Is character a line break? */
+  def isLineBreakChar(c: Char) = (c: @switch) match {
+    case LF|FF|CR|SU  => true
+    case _            => false
+  }
+
+  /** Is character a whitespace character (but not a new line)? */
+  def isWhitespace(c: Char) =
+    c == ' ' || c == '\t' || c == CR
+
+  /** Can character form part of a doc comment variable $xxx? */
+  def isVarPart(c: Char) =
+    '0' <= c && c <= '9' || 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z'
+
+  /** Can character start an alphanumeric Scala identifier? */
+  def isIdentifierStart(c: Char): Boolean =
+    (c == '_') || (c == '$') || Character.isUnicodeIdentifierStart(c)
+
+  /** Can character form part of an alphanumeric Scala identifier? */
+  def isIdentifierPart(c: Char) =
+    (c == '$') || Character.isUnicodeIdentifierPart(c)
+
+  /** Is character a math or other symbol in Unicode?  */
+  def isSpecial(c: Char) = {
+    val chtp = Character.getType(c)
+    chtp == Character.MATH_SYMBOL.toInt || chtp == Character.OTHER_SYMBOL.toInt
+  }
+
+  private final val otherLetters = Set[Char]('\u0024', '\u005F')  // '$' and '_'
+  private final val letterGroups = {
+    import JCharacter._
+    Set[Byte](LOWERCASE_LETTER, UPPERCASE_LETTER, OTHER_LETTER, TITLECASE_LETTER, LETTER_NUMBER)
+  }
+  def isScalaLetter(ch: Char) = letterGroups(JCharacter.getType(ch).toByte) || otherLetters(ch)
+
+  /** Can character form part of a Scala operator name? */
+  def isOperatorPart(c : Char) : Boolean = (c: @switch) match {
+    case '~' | '!' | '@' | '#' | '%' |
+         '^' | '*' | '+' | '-' | '<' |
+         '>' | '?' | ':' | '=' | '&' |
+         '|' | '/' | '\\' => true
+    case c => isSpecial(c)
+  }
+}
diff --git a/src/dotty/tools/dotc/util/FreshNameCreator.scala b/src/dotty/tools/dotc/util/FreshNameCreator.scala
new file mode 100644
index 000000000..d0c007d94
--- /dev/null
+++ b/src/dotty/tools/dotc/util/FreshNameCreator.scala
@@ -0,0 +1,38 @@
+package dotty.tools
+package dotc
+package util
+
+import scala.collection.mutable
+
+trait FreshNameCreator {
+  def newName(): String
+  def newName(prefix: String): String
+
+  @deprecated("use newName(prefix)", "2.9.0")
+  def newName(pos: scala.reflect.internal.util.Position, prefix: String): String = newName(prefix)
+  @deprecated("use newName()", "2.9.0")
+  def newName(pos: scala.reflect.internal.util.Position): String = newName()
+}
+
+object FreshNameCreator {
+  class Default extends FreshNameCreator {
+    protected var counter = 0
+    protected val counters = mutable.HashMap[String, Int]() withDefaultValue 0
+
+    /**
+     * Create a fresh name with the given prefix. It is guaranteed
+     * that the returned name has never been returned by a previous
+     * call to this function (provided the prefix does not end in a digit).
+     */
+    def newName(prefix: String): String = {
+      val safePrefix = prefix.replaceAll("""[<>]""", """\$""")
+      counters(safePrefix) += 1
+
+      safePrefix + counters(safePrefix)
+    }
+    def newName(): String = {
+      counter += 1
+      "$" + counter + "$"
+    }
+  }
+}
diff --git a/src/dotty/tools/dotc/util/Positions.scala b/src/dotty/tools/dotc/util/Positions.scala
new file mode 100644
index 000000000..31eb2ff0d
--- /dev/null
+++ b/src/dotty/tools/dotc/util/Positions.scala
@@ -0,0 +1,88 @@
+package dotty.tools.dotc
+package util
+
+/** Position format in little endian:
+ *  Start: unsigned 26 Bits (works for source files up to 64M)
+ *  End: unsigned 26 Bits
+ *  Point: unsigned 12 Bits relative to start
+ *  NoPosition encoded as -1L (this is a normally invalid position
+ *  because point would lie beyond end.
+ */
+object Positions {
+
+  private val StartEndBits = 26
+  private val StartEndMask = (1 << StartEndBits) - 1
+  private val PointOffsetLimit = 1L << (64 - StartEndBits * 2)
+
+  class Position(val coords: Long) extends AnyVal {
+    def point: Int = start + (coords >>> (StartEndBits * 2)).toInt
+    def start: Int = (coords & StartEndMask).toInt
+    def end: Int = ((coords >>> StartEndBits) & StartEndMask).toInt
+
+    /** The union of two positions. Tries to keep the point offset of
+     *  the first positon if within range. Otherwise, pointoffset will be 0.
+     */
+    def union(that: Position) =
+      if (!this.exists) that
+      else if (!that.exists) this
+      else {
+        val start = this.start min that.start
+        val end = this.end max that.end
+        var pointOffset = this.point - start
+        if (pointOffset >= PointOffsetLimit) pointOffset = 0
+        Position(start, end, pointOffset)
+      }
+
+    def exists = this != NoPosition
+
+    def shift(offset: Int) =
+      if (exists) Position(start + offset, end + offset, point - start)
+      else this
+
+    def focus = Position(point)
+
+    def withStart(start: Int) = Position(start, this.end, this.point - start)
+    def withEnd(end: Int) = Position(this.start, end, this.point - this.start)
+    def withPoint(point: Int) = Position(this.start, this.end, point - this.start)
+  }
+
+  def Position(start: Int, end: Int, pointOffset: Int = 0): Position =
+    new Position(
+      (start & StartEndMask).toLong |
+      ((end & StartEndMask).toLong << StartEndBits) |
+      (pointOffset.toLong << (StartEndBits * 2)))
+
+  def Position(point: Int): Position = Position(point, point, 0)
+
+  val NoPosition = new Position(-1L)
+
+  case class SourcePosition(source: SourceFile, pos: Position) {
+    def point: Int = pos.point
+    def start: Int = pos.start
+    def end: Int = pos.end
+    def exists = pos.exists
+  }
+
+  val NoSourcePosition = SourcePosition(NoSource, NoPosition)
+
+  /** The coordinate of a symbol. This is either an index or
+   *  a point position.
+   */
+  class Coord(val encoding: Int) extends AnyVal {
+    def isIndex = encoding > 0
+    def isPosition = encoding <= 0
+    def toIndex: Int = {
+      assert(isIndex)
+      encoding - 1
+    }
+    def toPosition = {
+      assert(isPosition)
+      if (this == NoCoord) NoPosition else Position(1 - encoding)
+    }
+  }
+
+  def indexCoord(n: Int) = new Coord(n + 1)
+  def positionCoord(n: Int) = new Coord(-(n + 1))
+
+  val NoCoord = new Coord(0)
+}
\ No newline at end of file
diff --git a/src/dotty/tools/dotc/util/SourceFile.scala b/src/dotty/tools/dotc/util/SourceFile.scala
new file mode 100644
index 000000000..a7a677560
--- /dev/null
+++ b/src/dotty/tools/dotc/util/SourceFile.scala
@@ -0,0 +1,109 @@
+package dotty.tools
+package dotc
+package util
+
+import scala.collection.mutable.ArrayBuffer
+import dotty.tools.io._
+import annotation.tailrec
+import java.util.regex.Pattern
+import java.io.IOException
+import Chars._
+import ScriptSourceFile._
+import Positions._
+
+object ScriptSourceFile {
+  private val headerPattern = Pattern.compile("""^(::)?!#.*(\r|\n|\r\n)""", Pattern.MULTILINE)
+  private val headerStarts  = List("#!", "::#!")
+
+  def apply(file: AbstractFile, content: Array[Char]) = {
+    /** Length of the script header from the given content, if there is one.
+     *  The header begins with "#!" or "::#!" and ends with a line starting
+     *  with "!#" or "::!#".
+     */
+    val headerLength =
+      if (headerStarts exists (content startsWith _)) {
+        val matcher = headerPattern matcher content.mkString
+        if (matcher.find) matcher.end
+        else throw new IOException("script file does not close its header with !# or ::!#")
+      } else 0
+    new SourceFile(file, content drop headerLength) {
+      override val underlying = new SourceFile(file, content)
+    }
+  }
+}
+
+case class SourceFile(file: AbstractFile, content: Array[Char]) {
+
+  def this(_file: AbstractFile)                 = this(_file, _file.toCharArray)
+  def this(sourceName: String, cs: Seq[Char])   = this(new VirtualFile(sourceName), cs.toArray)
+  def this(file: AbstractFile, cs: Seq[Char])   = this(file, cs.toArray)
+
+  override def equals(that : Any) = that match {
+    case that : SourceFile => file.path == that.file.path && start == that.start
+    case _ => false
+  }
+  override def hashCode = file.path.## + start.##
+
+  val length = content.length
+
+  /** true for all source files except `NoSource` */
+  def exists: Boolean = true
+
+  /** The underlying source file */
+  def underlying: SourceFile = this
+
+  /** The start of this file in the underlying source file */
+  def start = 0
+
+  def atPos(pos: Position): SourcePosition =
+    SourcePosition(underlying, pos)
+
+  def isSelfContained = underlying eq this
+
+  /** Map a position to a position in the underlying source file.
+   *  For regular source files, simply return the argument.
+   */
+  def positionInUltimateSource(position: SourcePosition): SourcePosition =
+    SourcePosition(underlying, position.pos shift start)
+
+  def isLineBreak(idx: Int) =
+    if (idx >= length) false else {
+      val ch = content(idx)
+      // don't identify the CR in CR LF as a line break, since LF will do.
+      if (ch == CR) (idx + 1 == length) || (content(idx + 1) != LF)
+      else isLineBreakChar(ch)
+    }
+
+  def calculateLineIndices(cs: Array[Char]) = {
+    val buf = new ArrayBuffer[Int]
+    buf += 0
+    for (i <- 0 until cs.length) if (isLineBreak(i)) buf += i + 1
+    buf += cs.length // sentinel, so that findLine below works smoother
+    buf.toArray
+  }
+  private lazy val lineIndices: Array[Int] = calculateLineIndices(content)
+
+  /** Map line to offset of first character in line */
+  def lineToOffset(index : Int): Int = lineIndices(index)
+
+  /** A cache to speed up offsetToLine searches to similar lines */
+  private var lastLine = 0
+
+  /** Convert offset to line in this source file
+   *  Lines are numbered from 0
+   */
+  def offsetToLine(offset: Int): Int = {
+    val lines = lineIndices
+    def findLine(lo: Int, hi: Int, mid: Int): Int =
+      if (offset < lines(mid)) findLine(lo, mid - 1, (lo + mid - 1) / 2)
+      else if (offset >= lines(mid + 1)) findLine(mid + 1, hi, (mid + 1 + hi) / 2)
+      else mid
+    lastLine = findLine(0, lines.length, lastLine)
+    lastLine
+  }
+}
+
+object NoSource extends SourceFile("<no source>", Nil) {
+  override def exists = false
+}
+