refined doc comments generation; refactored cod...

refined doc comments generation; refactored code into new Chars,
DocStrings classes in util. Added some more doc comments to collection
classes.

11 files changed, 421 insertions, 224 deletions

diff --git a/src/compiler/scala/tools/nsc/ast/DocComments.scala b/src/compiler/scala/tools/nsc/ast/DocComments.scala
index c89924ecb3..8dc1586400 100755
--- a/src/compiler/scala/tools/nsc/ast/DocComments.scala
+++ b/src/compiler/scala/tools/nsc/ast/DocComments.scala
@@ -9,6 +9,8 @@ package ast
 
 import symtab._
 import util.{Position, NoPosition}
+import util.DocStrings._
+import util.Chars._
 import scala.collection.mutable.{HashMap, ListBuffer, StringBuilder}
 
 /*
@@ -17,118 +19,82 @@ import scala.collection.mutable.{HashMap, ListBuffer, StringBuilder}
  */
 trait DocComments { self: SymbolTable =>
 
-  val docComments = new HashMap[Symbol, DocComment] // !!! todo: inherit from comment?
+  /** The raw doc comment map */
+  val docComments = new HashMap[Symbol, DocComment]
 
-  private val defs = new HashMap[Symbol, Map[String, String]] {
-    override def default(key: Symbol) = Map()
-  }
-
-  private def getDocComment(sym: Symbol): Option[DocComment] = {
-    docComments get sym match {
-      case None =>
-        mapFind(sym.allOverriddenSymbols)(docComments.get)
-      case someSym =>
-        someSym
-    }
-  }
-
-  private def mapFind[A, B](xs: Iterable[A])(f: A => Option[B]): Option[B] = {
-    var res: Option[B] = None
-    val it = xs.iterator
-    while (res.isEmpty && it.hasNext) {
-      res = f(it.next())
-    }
-    res
-  }
+  /** The raw doc comment of symbol `sym`, as it appears in the source text, "" if missing.
+   */
+  def rawDocComment(sym: Symbol): String =
+    docComments get sym map (_.raw) getOrElse ""
 
-  /** Return the javadoc format of doc comment string `s`, including wiki expansion
+  /** The position of the raw doc comment of symbol `sym`, or NoPosition if missing
+   *  If a symbol does not have a doc comment but some overridden version of it does,
+   *  the posititon of the doc comment of the overridden version is returned instead.
    */
-  def toJavaDoc(s: String): String = expandWiki(s)
+  def docCommentPos(sym: Symbol): Position =
+    getDocComment(sym) map (_.pos) getOrElse NoPosition
 
-  /** The raw doc comment of symbol `sym`, as it appears in the source text, "" if missing.
+  /** The raw doc comment of symbol `sym`, minus @usecase and @define sections, augmented by
+   *  missing sections of an inherited doc comment.
+   *  If a symbol does not have a doc comment but some overridden version of it does,
+   *  the doc comment of the overridden version is copied instead.
    */
-  def rawDocComment(sym: Symbol): String = getDocComment(sym) map (_.raw) getOrElse ""
+  def cookedDocComment(sym: Symbol): String = {
+    val ownComment = docComments get sym map (_.template) getOrElse ""
+    sym.allOverriddenSymbols.view map cookedDocComment find ("" !=) match {
+      case None =>
+        ownComment
+      case Some(superComment) =>
+        if (ownComment == "") superComment
+        else merge(superComment, ownComment, sym)
+    }
+  }
 
-  /** The doc comment of symbol `sym` after variable expansion, or "" if missing.
+  /** The cooked doc comment of symbol `sym` after variable expansion, or "" if missing.
+   *
    *  @param sym  The symbol for which doc comment is returned
    *  @param site The class for which doc comments are generated
+   *  @throws ExpansionLimitExceeded  when more than 10 successive expansions
+   *                                  of the same string are done, which is
+   *                                  interpreted as a recursive variable definition.
    */
   def expandedDocComment(sym: Symbol, site: Symbol): String =
-    getDocComment(sym) map (_.expanded(site)) getOrElse ""
+    expandVariables(cookedDocComment(sym), site)
 
-  /** The expanded doc comment of symbol `sym` after variable expansion, or "" if missing.
+  /** The cooked doc comment of symbol `sym` after variable expansion, or "" if missing.
    *  @param sym  The symbol for which doc comment is returned (site is always the containing class)
    */
-  def expandedDocComment(sym: Symbol): String = expandedDocComment(sym, sym.owner)
-
-  /** The position of the doc comment of symbol `sym`, or NoPosition if missing */
-  def docCommentPos(sym: Symbol): Position = getDocComment(sym) map (_.pos) getOrElse NoPosition
+  def expandedDocComment(sym: Symbol): String = expandedDocComment(sym, sym)
 
   /** The list of use cases of doc comment of symbol `sym` seen as a member of class
    *  `site`. Each use case consists of a synthetic symbol (which is entered nowhere else),
    *  and an expanded doc comment string.
+   *
+   *  @param sym  The symbol for which use cases are returned
+   *  @param site The class for which doc comments are generated
+   *  @throws ExpansionLimitExceeded  when more than 10 successive expansions
+   *                                  of the same string are done, which is
+   *                                  interpreted as a recursive variable definition.
    */
   def useCases(sym: Symbol, site: Symbol): List[(Symbol, String)] = {
     def getUseCases(dc: DocComment) = {
       for (uc <- dc.useCases; defn <- uc.expandedDefs(site)) yield
-        (defn, uc.comment.expanded(site))
+        (defn,
+         expandVariables(merge(cookedDocComment(sym), uc.comment.raw, defn, copyFirstPara = true), site))
     }
     getDocComment(sym) map getUseCases getOrElse List()
   }
 
-  def useCases(sym: Symbol): List[(Symbol, String)] = useCases(sym, sym.owner)
-
-  private val usecasePrefix = "@usecase "
-  private val definePrefix  = "@define "
-
-  /** Returns index of string `str` after `in` skipping longest
-   *  sequence of space and tab characters.
-   */
-  private def skipWhitespace(str: String, in: Int): Int = {
-    var idx = in
-    do {
-      idx += 1
-    } while (idx < str.length && ((str charAt idx) == ' ' || (str charAt idx) == '\t'))
-    idx
-  }
+  def useCases(sym: Symbol): List[(Symbol, String)] = useCases(sym, sym)
 
-  /** Returns index of string `str` after `in` skipping longest
-   *  sequence of space and tab characters, possibly also containing
-   *  a single `*' character.
+  /** Returns the javadoc format of doc comment string `s`, including wiki expansion
    */
-  private def skipLineLead(str: String, in: Int, end: Int): Int = {
-    val idx = skipWhitespace(str, in)
-    if (idx < end && (str charAt idx) == '*') skipWhitespace(str, idx)
-    else idx
-  }
-
-  /** Extracts variable name from a string, stripping any pair of surrounding braces */
-  private def variableName(str: String): String =
-    if (str.length >= 2 && (str charAt 0) == '{' && (str charAt (str.length - 1)) == '}')
-      str.substring(1, str.length - 1)
-    else
-      str
-
-  private def isVarPart(ch: Char) =
-    '0' <= ch && ch <= '9' || 'A' <= ch && ch <= 'Z' || 'a' <= ch && ch <= 'z'
-
-  /** Returns index following variable, or start index if no variable was recognized
-   */
-  private def skipVariable(str: String, start: Int): Int = {
-    var idx = start
-    if (idx < str.length && (str charAt idx) == '{') {
-      do idx += 1
-      while (idx < str.length && (str charAt idx) != '}')
-      if (idx < str.length) idx + 1 else start
-    } else {
-      while (idx < str.length && isVarPart(str charAt idx))
-        idx += 1
-      idx
-    }
-  }
+  def toJavaDoc(s: String): String = expandWiki(s)
 
   private val wikiReplacements = List(
     ("""(\n\s*\*?)(\s*\n)"""    .r, """$1 <p>$2"""),
+    ("""<([^\w/])"""            .r, """&lt;$1"""),
+    ("""([^\w/])>"""            .r, """$1&gt;"""),
     ("""\{\{\{(.*(?:\n.*)*)\}\}\}""".r, """<pre>$1</pre>"""),
     ("""`([^`]*)`"""            .r, """<code>$1</code>"""),
     ("""__([^_]*)__"""          .r, """<u>$1</u>"""),
@@ -136,9 +102,97 @@ trait DocComments { self: SymbolTable =>
     ("""'''([^']*)'''"""        .r, """<b>$1</b>"""),
     ("""\^([^^]*)\^"""          .r, """<sup>$1</sup>"""),
     (""",,([^,]*),,"""          .r, """<sub>$1</sub>"""))
-  private def expandWiki(str: String): String =
+
+  /** Returns just the wiki expansion (this would correspond to
+   *  a comment in the input format of the JavaDoc tool, modulo differences
+   *  in tags.)
+   */
+  def expandWiki(str: String): String =
     (str /: wikiReplacements) { (str1, regexRepl) => regexRepl._1 replaceAllIn(str1, regexRepl._2) }
 
+
+  private def getDocComment(sym: Symbol): Option[DocComment] = docComments get sym match {
+    case None => mapFind(sym.allOverriddenSymbols)(docComments get)
+    case some => some
+  }
+
+  private def mapFind[A, B](xs: Iterable[A])(f: A => Option[B]): Option[B] = {
+    var res: Option[B] = None
+    val it = xs.iterator
+    while (res.isEmpty && it.hasNext) {
+      res = f(it.next())
+    }
+    res
+  }
+
+  /** Merge elements of doccomment `src` into doc comment `dst` for symbol `sym`.
+   *  In detail:
+   *  1. If `copyFirstPara` is true, copy first paragraph
+   *  2. For all parameters of `sym` if there is no @param section
+   *     in `dst` for that parameter name, but there is one on `src`, copy that section.
+   *  3. If there is no @return section in `dst` but there is one in `src`, copy it.
+   */
+  def merge(src: String, dst: String, sym: Symbol, copyFirstPara: Boolean = false): String = {
+    val srcSections = tagIndex(src)
+    val dstSections = tagIndex(dst)
+    val srcParams = paramDocs(src, "@param", srcSections)
+    val dstParams = paramDocs(dst, "@param", dstSections)
+    val srcTParams = paramDocs(src, "@tparam", srcSections)
+    val dstTParams = paramDocs(dst, "@tparam", dstSections)
+    val out = new StringBuilder
+    var copied = 0
+
+    var tocopy = startTag(
+      dst,
+      dstSections dropWhile (sec =>
+        !startsWithTag(dst, sec, "@param") &&
+        !startsWithTag(dst, sec, "@tparam") &&
+        !startsWithTag(dst, sec, "@return")))
+
+    if (copyFirstPara) {
+      val eop = // end of first para, which is delimited by blank line, or tag, or end of comment
+        findNext(src, 0) (src.charAt(_) == '\n') min startTag(src, srcSections)
+      out append src.substring(0, eop)
+      copied = 3
+      tocopy = 3
+    }
+
+    def mergeSection(srcSec: Option[(Int, Int)], dstSec: Option[(Int, Int)]) = dstSec match {
+      case Some((start, end)) =>
+        if (end > tocopy) tocopy = end
+      case None =>
+        srcSec match {
+          case Some((start1, end1)) =>
+            out append dst.substring(copied, tocopy)
+            copied = tocopy
+            out append src.substring(start1, end1)
+          case None =>
+        }
+    }
+
+    def mergeParam(name: String, srcMap: Map[String, (Int, Int)], dstMap: Map[String, (Int, Int)]) =
+      mergeSection(srcMap get name, dstMap get name)
+
+    for (params <- sym.paramss; param <- params)
+      mergeSection(srcParams get param.name.toString, dstParams get param.name.toString)
+    for (tparam <- sym.typeParams)
+      mergeSection(srcTParams get tparam.name.toString, dstTParams get tparam.name.toString)
+    mergeSection(returnDoc(src, srcSections), returnDoc(dst, dstSections))
+
+    if (out.length == 0) dst
+    else {
+      out append dst.substring(copied)
+      out.toString
+    }
+  }
+
+  /** Maps symbols to the variable -> replacement maps that are defined
+   *  in their doc comments
+   */
+  private val defs = new HashMap[Symbol, Map[String, String]] {
+    override def default(key: Symbol) = Map()
+  }
+
   /** Lookup definition of variable.
    *
    *  @param vble  The variable for which a definition is searched
@@ -168,7 +222,7 @@ trait DocComments { self: SymbolTable =>
     if (expandCount < expandLimit) {
       try {
         val out = new StringBuilder
-        var start = 0
+        var copied = 0
         var idx = 0
         while (idx < str.length) {
           if ((str charAt idx) == '$') {
@@ -178,9 +232,9 @@ trait DocComments { self: SymbolTable =>
             if (vname.length > 0) {
               lookupVariable(vname, site) match {
                 case Some(replacement) =>
-                  out append str.substring(start, vstart)
+                  out append str.substring(copied, vstart)
                   out append replacement
-                  start = idx
+                  copied = idx
                 case None =>
                   //println("no replacement for "+vname) // !!!
               }
@@ -193,7 +247,7 @@ trait DocComments { self: SymbolTable =>
         }
         if (out.length == 0) str
         else {
-          out append str.substring(start)
+          out append str.substring(copied)
           expandVariables(out.toString, site)
         }
       } finally {
@@ -201,63 +255,45 @@ trait DocComments { self: SymbolTable =>
       }
     } else throw new ExpansionLimitExceeded(str)
 
+
+  // !!! todo: inherit from Comment?
   case class DocComment(raw: String, pos: Position = NoPosition) {
 
+    /** Returns:
+     *   template: the doc comment minus all @define and @usecase sections
+     *   defines : all define sections (as strings)
+     *   useCases: all usecase sections (as instances of class UseCase)
+     */
     lazy val (template, defines, useCases) = {
-      val parts = decompose(raw)
-      val (defines, usecases) = parts.tail partition (_._1 startsWith definePrefix)
-      val templ = parts.head._1
-      (templ,
-       defines map (_._1),
-       usecases map (decomposeUseCase(_, templ)))
-    }
-
-    def expanded(site: Symbol): String =
-      expandVariables(template, site)
+      val sections = tagIndex(raw, idx =>
+        startsWithTag(raw, idx, "@define") || startsWithTag(raw, idx, "@usecase"))
+      val (defines, usecases) = sections partition (startsWithTag(raw, _, "@define"))
+      val end = startTag(raw, sections)
 /*
-    expansions get site match {
-      case Some(str) => str
-      case None => val str =
-        expandVariables(template, sym, site)
-        expansions += (site -> str)
-        str
+      println("processing doc comment:")
+      println(raw)
+      println("===========>")
+      println(raw.substring(0, end))
+      println("++++++++++++++++")
+      println(sections map { case (s, e) => raw.substring(s, e) })
+      */
+      (if (end == raw.length - 2) raw else raw.substring(0, end) + "*/",
+       defines map { case (start, end) => raw.substring(start, end) },
+       usecases map { case (start, end) => decomposeUseCase(start, end) })
     }
-    private var expansions: Map[Symbol, String] = Map()
-*/
 
-
-    /** Decomposes a comment string into
-     *  an initial comment and a list of @define and @usecase clauses, each with starting index
-     */
-    private def decompose(str: String): List[(String, Int)] = {
-      val out = new ListBuffer[(String, Int)]
-      var segstart = 0
-      var idx = 3  // skip initial "/**"
-      val end = str.length - 2  // stop before final "*/"
-      var eolIdx = idx
-      while (idx < end) {
-        if ((str charAt idx) == '\n') {
-          eolIdx = idx
-          idx = skipLineLead(str, idx, end)
-          if (idx < end &&
-              (str charAt idx) == '@' &&
-              (str.startsWith(definePrefix, idx) || str.startsWith(usecasePrefix, idx))) {
-            var segment = str.substring(segstart, eolIdx)
-            if (segstart == 0) segment += "*/"
-            out += ((segment, segstart))
-            segstart = idx
-          }
-        } else idx += 1
-      }
-      if (segstart == 0)
-        List((str, 0))
-      else {
-        out += ((str.substring(segstart, eolIdx), segstart))
-        out.toList
-      }
+    private def decomposeUseCase(start: Int, end: Int): UseCase = {
+      val codeStart = skipWhitespace(raw, start + "@usecase".length)
+      val codeEnd = skipToEol(raw, codeStart)
+      val code = raw.substring(codeStart, codeEnd)
+      val codePos = subPos(codeStart, codeEnd)
+      val commentStart = skipLineLead(raw, codeEnd + 1) min end
+      val comment = "/** " + raw.substring(commentStart, end) + "*/"
+      val commentPos = subPos(commentStart, end)
+      UseCase(DocComment(comment, commentPos), code, codePos)
     }
 
-    def subPos(start: Int, end: Int) =
+    private def subPos(start: Int, end: Int) =
       if (pos == NoPosition) NoPosition
       else {
         val start1 = pos.start + start
@@ -265,29 +301,15 @@ trait DocComments { self: SymbolTable =>
         pos withStart start1 withPoint start1 withEnd end1
       }
 
-    def decomposeUseCase(stroff: (String, Int), mainComment: String): UseCase = {
-      val str = stroff._1
-      val offset = stroff._2
-      val start = usecasePrefix.length
-      var idx = start
-      while (idx < str.length && (str charAt idx) != '\n') idx += 1
-      val code = str.substring(start, idx)
-      val codePos = subPos(offset + usecasePrefix.length, offset + idx)
-      var firstParBreak = mainComment indexOf "<p>"
-      if (firstParBreak == -1) firstParBreak = mainComment.length - 2
-      val comment = mainComment.substring(0, firstParBreak)+"<p>"+
-        str.substring(skipLineLead(str, idx, str.length))+"*/"
-      val commentPos = subPos(offset + idx, offset + str.length)
-      UseCase(DocComment(comment, commentPos), code, codePos)
-    }
-
     def defineVariables(sym: Symbol) {
       for (str <- defines) {
-        val start = definePrefix.length
+        val start = skipWhitespace(str, "@define".length)
         var idx = skipVariable(str, start)
         val vble = variableName(str.substring(start, idx))
-        if (idx < str.length && (str charAt idx) == ' ') idx += 1
-        defs(sym) += vble -> str.substring(idx)
+        if (idx < str.length && isWhitespace(str charAt idx)) idx += 1
+        var end = str.lastIndexOf('\n')
+        if (end == -1) end = str.length
+        defs(sym) += vble -> str.substring(idx, end)
       }
       //if (defs(sym).nonEmpty) println("vars of "+sym+" = "+defs(sym))  // !!!
     }
@@ -307,21 +329,36 @@ trait DocComments { self: SymbolTable =>
       }
 
       def getSite(name: Name): Type = {
-        if (name == nme.this_) site.thisType
-        else {
-          def findIn(sites: List[Symbol]): Type = sites match {
-            case List() => NoType
-            case site :: sites1 => select(site.thisType, name, findIn(sites1))
-          }
-          val (classes, pkgs) = site.ownerChain.span(!_.isPackageClass)
-          findIn(classes ::: List(pkgs.head, definitions.RootClass))
+        def findIn(sites: List[Symbol]): Type = sites match {
+          case List() => NoType
+          case site :: sites1 => select(site.thisType, name, findIn(sites1))
         }
+        val (classes, pkgs) = site.ownerChain.span(!_.isPackageClass)
+        findIn(classes ::: List(pkgs.head, definitions.RootClass))
       }
 
       def getType(str: String): Type = {
-        val parts = str.split("""\.""").toList
+        def getParts(start: Int): List[String] = {
+          val end = skipIdent(str, start)
+          if (end == start) List()
+          else str.substring (start, end) :: {
+            if (end < str.length && (str charAt end) == '.') getParts(end + 1)
+            else List()
+          }
+        }
+        val parts = getParts(0)
         val partnames = (parts.init map newTermName) ::: List(newTypeName(parts.last))
-        (getSite(partnames.head) /: partnames.tail)(select(_, _, NoType))
+        val (start, rest) =
+          if (parts.head == "this")
+            (site.thisType, partnames.tail)
+          else if (parts.tail.nonEmpty && parts(1) == "this")
+            site.ownerChain.find(_.name.toString == parts.head) match {
+              case Some(clazz) => (clazz.thisType, partnames.drop(2))
+              case None => (NoType, List())
+            }
+          else
+            (getSite(partnames.head), partnames.tail)
+        (start /: rest)(select(_, _, NoType))
       }
 
       val aliasExpansions: List[Type] =
@@ -348,8 +385,8 @@ trait DocComments { self: SymbolTable =>
         def apply(tp: Type) = mapOver(tp) match {
           case tp1 @ TypeRef(pre, sym, args) if (sym.name.length > 1 && sym.name(0) == '$') =>
             subst(sym, aliases, aliasExpansions) match {
-              case TypeRef(pre, sym1, _) =>
-                TypeRef(pre, sym1, args)
+              case TypeRef(pre1, sym1, _) =>
+                TypeRef(pre1, sym1, args)
               case _ =>
                 tp1
             }
@@ -358,9 +395,10 @@ trait DocComments { self: SymbolTable =>
         }
       }
 
-      for (defn <- defined) yield
+      for (defn <- defined) yield {
         defn.cloneSymbol(site).setInfo(
           substAliases(defn.info).asSeenFrom(site.thisType, defn.owner))
+      }
     }
   }
 
diff --git a/src/compiler/scala/tools/nsc/ast/parser/MarkupParsers.scala b/src/compiler/scala/tools/nsc/ast/parser/MarkupParsers.scala
index d009ed71e8..67e1cd8304 100644
--- a/src/compiler/scala/tools/nsc/ast/parser/MarkupParsers.scala
+++ b/src/compiler/scala/tools/nsc/ast/parser/MarkupParsers.scala
@@ -13,7 +13,7 @@ import scala.util.control.ControlException
 import scala.tools.nsc.util.{Position,NoPosition,SourceFile,CharArrayReader}
 import scala.xml.{ Text, TextBuffer }
 import scala.xml.Utility.{ isNameStart, isNameChar, isSpace }
-import SourceFile.{ SU, LF }
+import util.Chars.{ SU, LF }
 import scala.annotation.switch
 
 // XXX/Note: many/most of the functions in here are almost direct cut and pastes
diff --git a/src/compiler/scala/tools/nsc/ast/parser/Scanners.scala b/src/compiler/scala/tools/nsc/ast/parser/Scanners.scala
index 3de13ed877..a9a3ded6b4 100644
--- a/src/compiler/scala/tools/nsc/ast/parser/Scanners.scala
+++ b/src/compiler/scala/tools/nsc/ast/parser/Scanners.scala
@@ -7,11 +7,12 @@ package scala.tools.nsc
 package ast.parser
 
 import scala.tools.nsc.util._
-import SourceFile.{LF, FF, CR, SU}
+import Chars.{LF, FF, CR, SU}
 import Tokens._
 import scala.annotation.switch
 import scala.collection.mutable.{ListBuffer, ArrayBuffer}
 import scala.xml.Utility.{ isNameStart }
+import util.Chars._
 
 trait Scanners {
   val global : Global
@@ -883,32 +884,6 @@ trait Scanners {
     }
   } // end Scanner
 
-  // ------------- character classification --------------------------------
-
-  def isIdentifierStart(c: Char): Boolean =
-    ('A' <= c && c <= 'Z') ||
-    ('a' <= c && c <= 'a') ||
-    (c == '_') || (c == '$') ||
-    Character.isUnicodeIdentifierStart(c)
-
-  def isIdentifierPart(c: Char) =
-    isIdentifierStart(c) ||
-    ('0' <= c && c <= '9') ||
-    Character.isUnicodeIdentifierPart(c)
-
-  def isSpecial(c: Char) = {
-    val chtp = Character.getType(c)
-    chtp == Character.MATH_SYMBOL.toInt || chtp == Character.OTHER_SYMBOL.toInt
-  }
-
-  def isOperatorPart(c : Char) : Boolean = (c: @switch) match {
-    case '~' | '!' | '@' | '#' | '%' |
-         '^' | '*' | '+' | '-' | '<' |
-         '>' | '?' | ':' | '=' | '&' |
-         '|' | '/' | '\\' => true
-    case c => isSpecial(c)
-  }
-
   // ------------- keyword configuration -----------------------------------
 
   /** Keyword array; maps from name indices to tokens */
diff --git a/src/compiler/scala/tools/nsc/ast/parser/Tokens.scala b/src/compiler/scala/tools/nsc/ast/parser/Tokens.scala
index b857675cc7..9aed4bd767 100644
--- a/src/compiler/scala/tools/nsc/ast/parser/Tokens.scala
+++ b/src/compiler/scala/tools/nsc/ast/parser/Tokens.scala
@@ -146,7 +146,7 @@ object Tokens {
   case ' ' | '\t' => true
   case _ => false
   }
-  import scala.tools.nsc.util.SourceFile._
+  import util.Chars._
 
   def isNewLine(at : Char) = at match {
   case CR | LF | FF => true
diff --git a/src/compiler/scala/tools/nsc/javac/JavaScanners.scala b/src/compiler/scala/tools/nsc/javac/JavaScanners.scala
index 2962e4bcba..113f1265e5 100644
--- a/src/compiler/scala/tools/nsc/javac/JavaScanners.scala
+++ b/src/compiler/scala/tools/nsc/javac/JavaScanners.scala
@@ -7,7 +7,7 @@ package scala.tools.nsc
 package javac
 
 import scala.tools.nsc.util._
-import SourceFile.{LF, FF, CR, SU}
+import Chars.{LF, FF, CR, SU}
 import JavaTokens._
 import scala.annotation.switch
 
diff --git a/src/compiler/scala/tools/nsc/javac/JavaTokens.scala b/src/compiler/scala/tools/nsc/javac/JavaTokens.scala
index 9889bcae7a..3e0637d374 100644
--- a/src/compiler/scala/tools/nsc/javac/JavaTokens.scala
+++ b/src/compiler/scala/tools/nsc/javac/JavaTokens.scala
@@ -149,7 +149,7 @@ object JavaTokens {
     case ' ' | '\t' => true
     case _ => false
   }
-  import scala.tools.nsc.util.SourceFile._
+  import util.Chars._
 
   def isNewLine(at : Char) = at match {
     case CR | LF | FF => true
diff --git a/src/compiler/scala/tools/nsc/util/CharArrayReader.scala b/src/compiler/scala/tools/nsc/util/CharArrayReader.scala
index 88175d26e4..a6c9edb8d7 100644
--- a/src/compiler/scala/tools/nsc/util/CharArrayReader.scala
+++ b/src/compiler/scala/tools/nsc/util/CharArrayReader.scala
@@ -7,7 +7,7 @@
 package scala.tools.nsc
 package util
 
-import scala.tools.nsc.util.SourceFile.{LF, FF, CR, SU}
+import Chars.{LF, FF, CR, SU}
 
 abstract class CharArrayReader { self =>
 
diff --git a/src/compiler/scala/tools/nsc/util/Chars.scala b/src/compiler/scala/tools/nsc/util/Chars.scala
new file mode 100755
index 0000000000..ce02b67633
--- /dev/null
+++ b/src/compiler/scala/tools/nsc/util/Chars.scala
@@ -0,0 +1,67 @@
+/* NSC -- new Scala compiler
+ * Copyright 2006-2010 LAMP/EPFL
+ * @author  Martin Odersky
+ */
+
+package scala.tools.nsc
+package util
+
+import annotation.{ tailrec, switch }
+
+/** Contains constants and classifier methods for characters */
+object Chars {
+
+  // Be very careful touching these.
+  // Apparently trivial changes to the way you write these constants
+  // will cause Scanners.scala to go from a nice efficient switch to
+  // a ghastly nested if statement which will bring the type checker
+  // to its knees. See ticket #1456
+  // Martin: (this should be verified now that the pattern rules have been redesigned).
+  final val LF = '\u000A'
+  final val FF = '\u000C'
+  final val CR = '\u000D'
+  final val SU = '\u001A'
+
+  /** Is character a line break? */
+  @inline def isLineBreakChar(c: Char) = (c: @switch) match {
+    case LF|FF|CR|SU  => true
+    case _            => false
+  }
+
+  /** Is character a whitespace character (but not a new line)? */
+  def isWhitespace(c: Char) =
+    c == ' ' || c == '\t' || c == CR
+
+  /** Can character form part of a doc comment variable $xxx? */
+  def isVarPart(c: Char) =
+    '0' <= c && c <= '9' || 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z'
+
+  /** Can character start an alphanumeric Scala identifier? */
+  def isIdentifierStart(c: Char): Boolean =
+    ('A' <= c && c <= 'Z') ||
+    ('a' <= c && c <= 'a') ||
+    (c == '_') || (c == '$') ||
+    Character.isUnicodeIdentifierStart(c)
+
+  /** Can character form part of an alphanumeric Scala identifier? */
+  def isIdentifierPart(c: Char) =
+    isIdentifierStart(c) ||
+    ('0' <= c && c <= '9') ||
+    Character.isUnicodeIdentifierPart(c)
+
+  /** Is character a math or other symbol in Unicode?  */
+  def isSpecial(c: Char) = {
+    val chtp = Character.getType(c)
+    chtp == Character.MATH_SYMBOL.toInt || chtp == Character.OTHER_SYMBOL.toInt
+  }
+
+  /** Can character form part of a Scala operator name? */
+  def isOperatorPart(c : Char) : Boolean = (c: @switch) match {
+    case '~' | '!' | '@' | '#' | '%' |
+         '^' | '*' | '+' | '-' | '<' |
+         '>' | '?' | ':' | '=' | '&' |
+         '|' | '/' | '\\' => true
+    case c => isSpecial(c)
+  }
+}
+
diff --git a/src/compiler/scala/tools/nsc/util/DocStrings.scala b/src/compiler/scala/tools/nsc/util/DocStrings.scala
new file mode 100755
index 0000000000..3392ef0577
--- /dev/null
+++ b/src/compiler/scala/tools/nsc/util/DocStrings.scala
@@ -0,0 +1,136 @@
+/* NSC -- new Scala compiler
+ * Copyright 2006-2010 LAMP/EPFL
+ * @author  Martin Odersky
+ */
+
+// $Id: ClassPath.scala 20028 2009-12-07 11:49:19Z cunei $
+
+package scala.tools.nsc
+package util
+
+import Chars._
+import scala.collection.mutable.{HashMap, ListBuffer, StringBuilder}
+
+/** Utilitity methods for doc comment strings
+ */
+object DocStrings {
+
+  /** Returns index of string `str` following `start` skipping longest
+   *  sequence of whitespace characters characters (but no newlines)
+   */
+  def skipWhitespace(str: String, start: Int): Int =
+    if (start < str.length && isWhitespace(str charAt start)) skipWhitespace(str, start + 1)
+    else start
+
+  /** Returns index of string `str` following `start` skipping
+   *  sequence of identifier characters.
+   */
+  def skipIdent(str: String, start: Int): Int =
+    if (start < str.length && isIdentifierPart(str charAt start)) skipIdent(str, start + 1)
+    else start
+
+  /** Returns index of string `str` after `start` skipping longest
+   *  sequence of space and tab characters, possibly also containing
+   *  a single `*' character.
+   *  @pre  start == str.length || str(start) == `\n'
+   */
+  def skipLineLead(str: String, start: Int): Int =
+    if (start == str.length) start
+    else {
+      val idx = skipWhitespace(str, start + 1)
+      if (idx < str.length && (str charAt idx) == '*') skipWhitespace(str, idx + 1)
+      else idx
+    }
+
+  /** Skips to next occurrence of `\n' following index `start`.
+   */
+  def skipToEol(str: String, start: Int): Int =
+    if (start < str.length && (str charAt start) != '\n') skipToEol(str, start + 1)
+    else start
+
+  /** Returns first index following `start` and starting a line (i.e. after skipLineLead)
+   *  which satisfies predicate `p'.
+   */
+  def findNext(str: String, start: Int)(p: Int => Boolean): Int = {
+    val idx = skipLineLead(str, skipToEol(str, start))
+    if (idx < str.length && !p(idx)) findNext(str, idx)(p)
+    else idx
+  }
+
+  /** Return first index following `start` and starting a line (i.e. after skipLineLead)
+   *  which satisfies predicate `p'.
+   */
+  def findAll(str: String, start: Int)(p: Int => Boolean): List[Int] = {
+    val idx = findNext(str, start)(p)
+    if (idx == str.length) List()
+    else idx :: findAll(str, idx)(p)
+  }
+
+  /** Produces a string index, which is a list of ``sections'', i.e
+   *  pairs of start/end positions of all tagged sections in the string.
+   *  Every section starts with a `@' and extends to the next `@', or
+   *  to the end of the comment string, but excluding the final two
+   *  charcters which terminate the comment.
+   */
+  def tagIndex(str: String, p: Int => Boolean = (idx => true)): List[(Int, Int)] =
+    findAll(str, 0) (idx => str(idx) == '@' && p(idx)) match {
+      case List() => List()
+      case idxs => idxs zip (idxs.tail ::: List(str.length - 2))
+    }
+
+  /** Does interval `iv` start with given `tag`?
+   */
+  def startsWithTag(str: String, section: (Int, Int), tag: String): Boolean =
+    startsWithTag(str, section._1, tag)
+
+  def startsWithTag(str: String, start: Int, tag: String): Boolean =
+    str.startsWith(tag, start) && !isIdentifierPart(str charAt (start + tag.length))
+
+
+  /** The first start tag of a list of tag intervals,
+   *  or the end of the whole comment string - 2 if list is empty
+   */
+  def startTag(str: String, sections: List[(Int, Int)]) = sections match {
+    case List() => str.length - 2
+    case (start, _) :: _ => start
+  }
+
+  /** A map from parameter names to start/end indices describing all parameter
+   *  sections in `str` tagged with `tag`, where `sections` is the index of `str`.
+   */
+  def paramDocs(str: String, tag: String, sections: List[(Int, Int)]): Map[String, (Int, Int)] =
+    Map() ++ {
+      for (section <- sections if startsWithTag(str, section, tag)) yield {
+        val start = skipWhitespace(str, section._1 + tag.length)
+        str.substring(start, skipIdent(str, start)) -> section
+      }
+    }
+
+  /** Optionally start and end index of return section in `str`, or `None`
+   *  if `str` does not have a @return.
+   */
+  def returnDoc(str: String, sections: List[(Int, Int)]): Option[(Int, Int)] =
+    sections find (startsWithTag(str, _, "@return"))
+
+  /** Extracts variable name from a string, stripping any pair of surrounding braces */
+  def variableName(str: String): String =
+    if (str.length >= 2 && (str charAt 0) == '{' && (str charAt (str.length - 1)) == '}')
+      str.substring(1, str.length - 1)
+    else
+      str
+
+  /** Returns index following variable, or start index if no variable was recognized
+   */
+  def skipVariable(str: String, start: Int): Int = {
+    var idx = start
+    if (idx < str.length && (str charAt idx) == '{') {
+      do idx += 1
+      while (idx < str.length && (str charAt idx) != '}')
+      if (idx < str.length) idx + 1 else start
+    } else {
+      while (idx < str.length && isVarPart(str charAt idx))
+        idx += 1
+      idx
+    }
+  }
+}
diff --git a/src/compiler/scala/tools/nsc/util/JavaCharArrayReader.scala b/src/compiler/scala/tools/nsc/util/JavaCharArrayReader.scala
index 2a6080f96d..731e04e4dc 100644
--- a/src/compiler/scala/tools/nsc/util/JavaCharArrayReader.scala
+++ b/src/compiler/scala/tools/nsc/util/JavaCharArrayReader.scala
@@ -7,7 +7,7 @@
 package scala.tools.nsc
 package util
 
-import scala.tools.nsc.util.SourceFile.{LF, FF, CR, SU}
+import Chars.{LF, FF, CR, SU}
 
 class JavaCharArrayReader(buf: IndexedSeq[Char], start: Int, /* startline: int, startcol: int, */
                       decodeUni: Boolean, error: String => Unit) extends Iterator[Char] with Cloneable {
diff --git a/src/compiler/scala/tools/nsc/util/SourceFile.scala b/src/compiler/scala/tools/nsc/util/SourceFile.scala
index f23f224eac..e8b6a1c63c 100644
--- a/src/compiler/scala/tools/nsc/util/SourceFile.scala
+++ b/src/compiler/scala/tools/nsc/util/SourceFile.scala
@@ -7,29 +7,14 @@
 
 package scala.tools.nsc
 package util
+
 import scala.tools.nsc.io.{AbstractFile, VirtualFile}
 import scala.collection.mutable.ArrayBuffer
 import annotation.{ tailrec, switch }
+import Chars._
 
-object SourceFile {
-  // Be very careful touching these.
-  // Apparently trivial changes to the way you write these constants
-  // will cause Scanners.scala to go from a nice efficient switch to
-  // a ghastly nested if statement which will bring the type checker
-  // to its knees. See ticket #1456
-  final val LF = '\u000A'
-  final val FF = '\u000C'
-  final val CR = '\u000D'
-  final val SU = '\u001A'
-
-  @inline def isLineBreakChar(c: Char) = (c: @switch) match {
-    case LF|FF|CR|SU  => true
-    case _            => false
-  }
-}
 /** abstract base class of a source file used in the compiler */
 abstract class SourceFile {
-  import SourceFile._
   def content : Array[Char]         // normalized, must end in SU
   def file    : AbstractFile
   def isLineBreak(idx : Int) : Boolean
@@ -64,7 +49,6 @@ abstract class SourceFile {
 
 /** a file whose contents do not change over time */
 class BatchSourceFile(val file : AbstractFile, val content: Array[Char]) extends SourceFile {
-  import SourceFile._
 
   def this(_file: AbstractFile)                 = this(_file, _file.toCharArray)
   def this(sourceName: String, cs: Seq[Char])   = this(new VirtualFile(sourceName), cs.toArray)
@@ -84,10 +68,7 @@ class BatchSourceFile(val file : AbstractFile, val content: Array[Char]) extends
 
   override def identifier(pos: Position, compiler: Global) =
     if (pos.isDefined && pos.source == this && pos.point != -1) {
-      def isOK(c: Char) = {
-        import compiler.syntaxAnalyzer.{ isOperatorPart, isIdentifierPart }
-        isIdentifierPart(c) || isOperatorPart(c)
-      }
+      def isOK(c: Char) = isIdentifierPart(c) || isOperatorPart(c)
       Some(new String(content drop pos.point takeWhile isOK))
     } else {
       super.identifier(pos, compiler)
@@ -211,7 +192,7 @@ extends BatchSourceFile(name, contents)
 
 object CompoundSourceFile {
   private[util] def stripSU(chars: Array[Char]) =
-    if (chars.length > 0 && chars.last == SourceFile.SU)
+    if (chars.length > 0 && chars.last == SU)
       chars.slice(0, chars.length-1)
     else
       chars