Move docstring cooking to dotty

4 files changed, 604 insertions, 21 deletions

diff --git a/src/dotty/tools/dotc/core/Comments.scala b/src/dotty/tools/dotc/core/Comments.scala
index ea5726fa0..3e562baff 100644
--- a/src/dotty/tools/dotc/core/Comments.scala
+++ b/src/dotty/tools/dotc/core/Comments.scala
@@ -9,24 +9,38 @@ import Contexts.Context
 import Flags.EmptyFlags
 import dotc.util.SourceFile
 import dotc.util.Positions._
+import dotc.util.CommentParsing._
 import dotc.parsing.Parsers.Parser
-import dotty.tools.dottydoc.model.comment.CommentUtils._
 
 object Comments {
 
-  case class Comment(pos: Position, raw: String)(implicit ctx: Context) {
+  abstract case class Comment(pos: Position, raw: String)(implicit ctx: Context) { self =>
+    def isExpanded: Boolean
+
+    def usecases: List[UseCase]
 
     val isDocComment = raw.startsWith("/**")
 
-    private[this] lazy val sections = tagIndex(raw)
+    def expand(f: String => String): Comment = new Comment(pos, f(raw)) {
+      val isExpanded = true
+      val usecases = self.usecases
+    }
 
-    private def fold[A](z: A)(op: => A) = if (!isDocComment) z else op
+    def withUsecases: Comment = new Comment(pos, stripUsecases) {
+      val isExpanded = self.isExpanded
+      val usecases = parseUsecases
+    }
 
-    lazy val usecases = fold(List.empty[UseCase]) {
-      sections
+    private[this] lazy val stripUsecases: String =
+      removeSections(raw, "@usecase", "@define")
+
+    private[this] lazy val parseUsecases: List[UseCase] =
+      if (!raw.startsWith("/**"))
+        List.empty[UseCase]
+      else
+        tagIndex(raw)
         .filter { startsWithTag(raw, _, "@usecase") }
         .map { case (start, end) => decomposeUseCase(start, end) }
-    }
 
     /** Turns a usecase section into a UseCase, with code changed to:
      *  {{{
@@ -36,7 +50,15 @@ object Comments {
      *  def foo: A = ???
      *  }}}
      */
-    private def decomposeUseCase(start: Int, end: Int): UseCase = {
+    private[this] def decomposeUseCase(start: Int, end: Int): UseCase = {
+      def subPos(start: Int, end: Int) =
+        if (pos == NoPosition) NoPosition
+        else {
+          val start1 = pos.start + start
+          val end1 = pos.end + end
+          pos withStart start1 withPoint start1 withEnd end1
+        }
+
       val codeStart    = skipWhitespace(raw, start + "@usecase".length)
       val codeEnd      = skipToEol(raw, codeStart)
       val code         = raw.substring(codeStart, codeEnd) + " = ???"
@@ -47,13 +69,13 @@ object Comments {
 
       UseCase(Comment(commentPos, commentStr), code, codePos)
     }
+  }
 
-    private def subPos(start: Int, end: Int) =
-      if (pos == NoPosition) NoPosition
-      else {
-        val start1 = pos.start + start
-        val end1 = pos.end + end
-        pos withStart start1 withPoint start1 withEnd end1
+  object Comment {
+    def apply(pos: Position, raw: String, expanded: Boolean = false, usc: List[UseCase] = Nil)(implicit ctx: Context): Comment =
+      new Comment(pos, raw) {
+        val isExpanded = expanded
+        val usecases = usc
       }
   }
 
@@ -74,4 +96,328 @@ object Comments {
       }
     }
   }
+
+  /**
+   * Port of DocComment.scala from nsc
+   * @author Martin Odersky
+   * @author Felix Mulder
+   */
+  class CommentExpander {
+    import dotc.config.Printers.dottydoc
+    import scala.collection.mutable
+
+    def expand(sym: Symbol, site: Symbol)(implicit ctx: Context): String = {
+      val parent = if (site != NoSymbol) site else sym
+      defineVariables(parent)
+      expandedDocComment(sym, parent)
+    }
+
+    /** The cooked doc comment of symbol `sym` after variable expansion, or "" if missing.
+     *
+     *  @param sym  The symbol for which doc comment is returned
+     *  @param site The class for which doc comments are generated
+     *  @throws ExpansionLimitExceeded  when more than 10 successive expansions
+     *                                  of the same string are done, which is
+     *                                  interpreted as a recursive variable definition.
+     */
+    def expandedDocComment(sym: Symbol, site: Symbol, docStr: String = "")(implicit ctx: Context): String = {
+      // when parsing a top level class or module, use the (module-)class itself to look up variable definitions
+      val parent = if ((sym.is(Flags.Module) || sym.isClass) && site.is(Flags.Package)) sym
+                   else site
+      expandVariables(cookedDocComment(sym, docStr), sym, parent)
+    }
+
+    private def template(raw: String): String =
+      removeSections(raw, "@define")
+
+    private def defines(raw: String): List[String] = {
+      val sections = tagIndex(raw)
+      val defines = sections filter { startsWithTag(raw, _, "@define") }
+      val usecases = sections filter { startsWithTag(raw, _, "@usecase") }
+      val end = startTag(raw, (defines ::: usecases).sortBy(_._1))
+
+      defines map { case (start, end) => raw.substring(start, end) }
+    }
+
+    private def replaceInheritDocToInheritdoc(docStr: String): String  =
+      docStr.replaceAll("""\{@inheritDoc\p{Zs}*\}""", "@inheritdoc")
+
+    /** The cooked doc comment of an overridden symbol */
+    protected def superComment(sym: Symbol)(implicit ctx: Context): Option[String] =
+      allInheritedOverriddenSymbols(sym).iterator map (x => cookedDocComment(x)) find (_ != "")
+
+    private val cookedDocComments = mutable.HashMap[Symbol, String]()
+
+    /** The raw doc comment of symbol `sym`, minus usecase and define sections, augmented by
+     *  missing sections of an inherited doc comment.
+     *  If a symbol does not have a doc comment but some overridden version of it does,
+     *  the doc comment of the overridden version is copied instead.
+     */
+    def cookedDocComment(sym: Symbol, docStr: String = "")(implicit ctx: Context): String = cookedDocComments.getOrElseUpdate(sym, {
+      var ownComment =
+        if (docStr.length == 0) ctx.docbase.docstring(sym).map(c => template(c.raw)).getOrElse("")
+        else template(docStr)
+      ownComment = replaceInheritDocToInheritdoc(ownComment)
+
+      superComment(sym) match {
+        case None =>
+          // SI-8210 - The warning would be false negative when this symbol is a setter
+          if (ownComment.indexOf("@inheritdoc") != -1 && ! sym.isSetter)
+            dottydoc.println(s"${sym.pos}: the comment for ${sym} contains @inheritdoc, but no parent comment is available to inherit from.")
+          ownComment.replaceAllLiterally("@inheritdoc", "<invalid inheritdoc annotation>")
+        case Some(sc) =>
+          if (ownComment == "") sc
+          else expandInheritdoc(sc, merge(sc, ownComment, sym), sym)
+      }
+    })
+
+    private def isMovable(str: String, sec: (Int, Int)): Boolean =
+      startsWithTag(str, sec, "@param") ||
+      startsWithTag(str, sec, "@tparam") ||
+      startsWithTag(str, sec, "@return")
+
+    def merge(src: String, dst: String, sym: Symbol, copyFirstPara: Boolean = false): String = {
+      val srcSections  = tagIndex(src)
+      val dstSections  = tagIndex(dst)
+      val srcParams    = paramDocs(src, "@param", srcSections)
+      val dstParams    = paramDocs(dst, "@param", dstSections)
+      val srcTParams   = paramDocs(src, "@tparam", srcSections)
+      val dstTParams   = paramDocs(dst, "@tparam", dstSections)
+      val out          = new StringBuilder
+      var copied       = 0
+      var tocopy       = startTag(dst, dstSections dropWhile (!isMovable(dst, _)))
+
+      if (copyFirstPara) {
+        val eop = // end of comment body (first para), which is delimited by blank line, or tag, or end of comment
+          (findNext(src, 0)(src.charAt(_) == '\n')) min startTag(src, srcSections)
+        out append src.substring(0, eop).trim
+        copied = 3
+        tocopy = 3
+      }
+
+      def mergeSection(srcSec: Option[(Int, Int)], dstSec: Option[(Int, Int)]) = dstSec match {
+        case Some((start, end)) =>
+          if (end > tocopy) tocopy = end
+        case None =>
+          srcSec match {
+            case Some((start1, end1)) => {
+              out append dst.substring(copied, tocopy).trim
+              out append "\n"
+              copied = tocopy
+              out append src.substring(start1, end1).trim
+            }
+            case None =>
+          }
+      }
+
+      //TODO: enable this once you know how to get `sym.paramss`
+      /*
+      for (params <- sym.paramss; param <- params)
+        mergeSection(srcParams get param.name.toString, dstParams get param.name.toString)
+      for (tparam <- sym.typeParams)
+        mergeSection(srcTParams get tparam.name.toString, dstTParams get tparam.name.toString)
+
+      mergeSection(returnDoc(src, srcSections), returnDoc(dst, dstSections))
+      mergeSection(groupDoc(src, srcSections), groupDoc(dst, dstSections))
+      */
+
+      if (out.length == 0) dst
+      else {
+        out append dst.substring(copied)
+        out.toString
+      }
+    }
+
+    /**
+     * Expand inheritdoc tags
+     *  - for the main comment we transform the inheritdoc into the super variable,
+     *  and the variable expansion can expand it further
+     *  - for the param, tparam and throws sections we must replace comments on the spot
+     *
+     * This is done separately, for two reasons:
+     * 1. It takes longer to run compared to merge
+     * 2. The inheritdoc annotation should not be used very often, as building the comment from pieces severely
+     * impacts performance
+     *
+     * @param parent The source (or parent) comment
+     * @param child  The child (overriding member or usecase) comment
+     * @param sym    The child symbol
+     * @return       The child comment with the inheritdoc sections expanded
+     */
+    def expandInheritdoc(parent: String, child: String, sym: Symbol): String =
+      if (child.indexOf("@inheritdoc") == -1)
+        child
+      else {
+        val parentSections    = tagIndex(parent)
+        val childSections     = tagIndex(child)
+        val parentTagMap      = sectionTagMap(parent, parentSections)
+        val parentNamedParams = Map() +
+          ("@param"  -> paramDocs(parent, "@param", parentSections)) +
+          ("@tparam" -> paramDocs(parent, "@tparam", parentSections)) +
+          ("@throws" -> paramDocs(parent, "@throws", parentSections))
+
+        val out         = new StringBuilder
+
+        def replaceInheritdoc(childSection: String, parentSection: => String) =
+          if (childSection.indexOf("@inheritdoc") == -1)
+            childSection
+          else
+            childSection.replaceAllLiterally("@inheritdoc", parentSection)
+
+        def getParentSection(section: (Int, Int)): String = {
+
+          def getSectionHeader = extractSectionTag(child, section) match {
+            case param@("@param"|"@tparam"|"@throws")  => param + " "  + extractSectionParam(child, section)
+            case other     => other
+          }
+
+          def sectionString(param: String, paramMap: Map[String, (Int, Int)]): String =
+            paramMap.get(param) match {
+              case Some(section) =>
+                // Cleanup the section tag and parameter
+                val sectionTextBounds = extractSectionText(parent, section)
+                cleanupSectionText(parent.substring(sectionTextBounds._1, sectionTextBounds._2))
+              case None =>
+                dottydoc.println(s"""${sym.pos}: the """" + getSectionHeader + "\" annotation of the " + sym +
+                    " comment contains @inheritdoc, but the corresponding section in the parent is not defined.")
+                "<invalid inheritdoc annotation>"
+            }
+
+          child.substring(section._1, section._1 + 7) match {
+            case param@("@param "|"@tparam"|"@throws") =>
+              sectionString(extractSectionParam(child, section), parentNamedParams(param.trim))
+            case _                                     =>
+              sectionString(extractSectionTag(child, section), parentTagMap)
+          }
+        }
+
+        def mainComment(str: String, sections: List[(Int, Int)]): String =
+          if (str.trim.length > 3)
+            str.trim.substring(3, startTag(str, sections))
+          else
+            ""
+
+        // Append main comment
+        out.append("/**")
+        out.append(replaceInheritdoc(mainComment(child, childSections), mainComment(parent, parentSections)))
+
+        // Append sections
+        for (section <- childSections)
+          out.append(replaceInheritdoc(child.substring(section._1, section._2), getParentSection(section)))
+
+        out.append("*/")
+        out.toString
+      }
+
+    protected def expandVariables(initialStr: String, sym: Symbol, site: Symbol)(implicit ctx: Context): String = {
+      val expandLimit = 10
+
+      def expandInternal(str: String, depth: Int): String = {
+        if (depth >= expandLimit)
+          throw new ExpansionLimitExceeded(str)
+
+        val out         = new StringBuilder
+        var copied, idx = 0
+        // excluding variables written as \$foo so we can use them when
+        // necessary to document things like Symbol#decode
+        def isEscaped = idx > 0 && str.charAt(idx - 1) == '\\'
+        while (idx < str.length) {
+          if ((str charAt idx) != '$' || isEscaped)
+            idx += 1
+          else {
+            val vstart = idx
+            idx = skipVariable(str, idx + 1)
+            def replaceWith(repl: String) = {
+              out append str.substring(copied, vstart)
+              out append repl
+              copied = idx
+            }
+            variableName(str.substring(vstart + 1, idx)) match {
+              case "super"    =>
+                superComment(sym) foreach { sc =>
+                  val superSections = tagIndex(sc)
+                  replaceWith(sc.substring(3, startTag(sc, superSections)))
+                  for (sec @ (start, end) <- superSections)
+                    if (!isMovable(sc, sec)) out append sc.substring(start, end)
+                }
+              case "" => idx += 1
+              case vname  =>
+                lookupVariable(vname, site) match {
+                  case Some(replacement) => replaceWith(replacement)
+                  case None              =>
+                    dottydoc.println(s"Variable $vname undefined in comment for $sym in $site")
+                }
+              }
+          }
+        }
+        if (out.length == 0) str
+        else {
+          out append str.substring(copied)
+          expandInternal(out.toString, depth + 1)
+        }
+      }
+
+      // We suppressed expanding \$ throughout the recursion, and now we
+      // need to replace \$ with $ so it looks as intended.
+      expandInternal(initialStr, 0).replaceAllLiterally("""\$""", "$")
+    }
+
+    def defineVariables(sym: Symbol)(implicit ctx: Context) = {
+      val Trim = "(?s)^[\\s&&[^\n\r]]*(.*?)\\s*$".r
+
+      val raw = ctx.docbase.docstring(sym).map(_.raw).getOrElse("")
+      defs(sym) ++= defines(raw).map {
+        str => {
+          val start = skipWhitespace(str, "@define".length)
+          val (key, value) = str.splitAt(skipVariable(str, start))
+          key.drop(start) -> value
+        }
+      } map {
+        case (key, Trim(value)) =>
+          variableName(key) -> value.replaceAll("\\s+\\*+$", "")
+      }
+    }
+
+    /** Maps symbols to the variable -> replacement maps that are defined
+     *  in their doc comments
+     */
+    private val defs = mutable.HashMap[Symbol, Map[String, String]]() withDefaultValue Map()
+
+    /** Lookup definition of variable.
+     *
+     *  @param vble  The variable for which a definition is searched
+     *  @param site  The class for which doc comments are generated
+     */
+    def lookupVariable(vble: String, site: Symbol)(implicit ctx: Context): Option[String] = site match {
+      case NoSymbol => None
+      case _        =>
+        val searchList =
+          if (site.flags.is(Flags.Module)) site :: site.info.baseClasses
+          else site.info.baseClasses
+
+        searchList collectFirst { case x if defs(x) contains vble => defs(x)(vble) } match {
+          case Some(str) if str startsWith "$" => lookupVariable(str.tail, site)
+          case res                             => res orElse lookupVariable(vble, site.owner)
+        }
+    }
+
+    /** The position of the raw doc comment of symbol `sym`, or NoPosition if missing
+     *  If a symbol does not have a doc comment but some overridden version of it does,
+     *  the position of the doc comment of the overridden version is returned instead.
+     */
+    def docCommentPos(sym: Symbol)(implicit ctx: Context): Position =
+      ctx.docbase.docstring(sym).map(_.pos).getOrElse(NoPosition)
+
+    /** A version which doesn't consider self types, as a temporary measure:
+     *  an infinite loop has broken out between superComment and cookedDocComment
+     *  since r23926.
+     */
+    private def allInheritedOverriddenSymbols(sym: Symbol)(implicit ctx: Context): List[Symbol] = {
+      if (!sym.owner.isClass) Nil
+      else sym.allOverriddenSymbols.toList.filter(_ != NoSymbol) //TODO: could also be `sym.owner.allOverrid..`
+      //else sym.owner.ancestors map (sym overriddenSymbol _) filter (_ != NoSymbol)
+    }
+
+    class ExpansionLimitExceeded(str: String) extends Exception
+  }
 }
diff --git a/src/dotty/tools/dotc/core/Contexts.scala b/src/dotty/tools/dotc/core/Contexts.scala
index a37e367fa..3378f0790 100644
--- a/src/dotty/tools/dotc/core/Contexts.scala
+++ b/src/dotty/tools/dotc/core/Contexts.scala
@@ -582,6 +582,8 @@ object Contexts {
     private[this] val _docstrings: mutable.Map[Symbol, Comment] =
       mutable.Map.empty
 
+    val templateExpander = new CommentExpander
+
     def docstrings: Map[Symbol, Comment] = _docstrings.toMap
 
     def docstring(sym: Symbol): Option[Comment] = _docstrings.get(sym)
diff --git a/src/dotty/tools/dotc/typer/Typer.scala b/src/dotty/tools/dotc/typer/Typer.scala
index 2a8e7da9a..b24fc6237 100644
--- a/src/dotty/tools/dotc/typer/Typer.scala
+++ b/src/dotty/tools/dotc/typer/Typer.scala
@@ -1538,7 +1538,7 @@ class Typer extends Namer with TypeAssigner with Applications with Implicits wit
     tpd.cpy.DefDef(mdef)(rhs = Inliner.bodyToInline(mdef.symbol)) ::
         Inliner.removeInlineAccessors(mdef.symbol)
 
-  def typedUsecases(syms: List[Symbol], owner: Symbol)(implicit ctx: Context): Unit = {
+  private def typedUsecases(syms: List[Symbol], owner: Symbol)(implicit ctx: Context): Unit = {
     val relevantSyms = syms.filter(ctx.docbase.docstring(_).isDefined)
     relevantSyms.foreach { sym =>
       expandParentDocs(sym)
@@ -1555,13 +1555,16 @@ class Typer extends Namer with TypeAssigner with Applications with Implicits wit
     }
   }
 
-  import dotty.tools.dottydoc.model.comment.CommentExpander
-  val expander = new CommentExpander {}
-  def expandParentDocs(sym: Symbol)(implicit ctx: Context): Unit =
+  private def expandParentDocs(sym: Symbol)(implicit ctx: Context): Unit =
     ctx.docbase.docstring(sym).foreach { cmt =>
-      def expandDoc(owner: Symbol): Unit = {
-        expander.defineVariables(sym)
-        val newCmt = Comment(cmt.pos, expander.expandedDocComment(sym, owner, cmt.raw))
+      def expandDoc(owner: Symbol): Unit = if (!cmt.isExpanded) {
+        val tplExp = ctx.docbase.templateExpander
+        tplExp.defineVariables(sym)
+
+        val newCmt = cmt
+          .expand(tplExp.expandedDocComment(sym, owner, _))
+          .withUsecases
+
         ctx.docbase.addDocstring(sym, Some(newCmt))
       }
 
diff --git a/src/dotty/tools/dotc/util/CommentParsing.scala b/src/dotty/tools/dotc/util/CommentParsing.scala
new file mode 100644
index 000000000..077776b5d
--- /dev/null
+++ b/src/dotty/tools/dotc/util/CommentParsing.scala
@@ -0,0 +1,232 @@
+/*
+ * Port of DocStrings.scala from nsc
+ * @author Martin Odersky
+ * @author Felix Mulder
+ */
+package dotty.tools.dotc.util
+
+object CommentParsing {
+  import scala.reflect.internal.Chars._
+
+  /** Returns index of string `str` following `start` skipping longest
+   *  sequence of whitespace characters characters (but no newlines)
+   */
+  def skipWhitespace(str: String, start: Int): Int =
+    if (start < str.length && isWhitespace(str charAt start)) skipWhitespace(str, start + 1)
+    else start
+
+  /** Returns index of string `str` following `start` skipping
+   *  sequence of identifier characters.
+   */
+  def skipIdent(str: String, start: Int): Int =
+    if (start < str.length && isIdentifierPart(str charAt start)) skipIdent(str, start + 1)
+    else start
+
+  /** Returns index of string `str` following `start` skipping
+   *  sequence of identifier characters.
+   */
+  def skipTag(str: String, start: Int): Int =
+    if (start < str.length && (str charAt start) == '@') skipIdent(str, start + 1)
+    else start
+
+
+  /** Returns index of string `str` after `start` skipping longest
+   *  sequence of space and tab characters, possibly also containing
+   *  a single `*` character or the `/``**` sequence.
+   *  @pre  start == str.length || str(start) == `\n`
+   */
+  def skipLineLead(str: String, start: Int): Int =
+    if (start == str.length) start
+    else {
+      val idx = skipWhitespace(str, start + 1)
+      if (idx < str.length && (str charAt idx) == '*') skipWhitespace(str, idx + 1)
+      else if (idx + 2 < str.length && (str charAt idx) == '/' && (str charAt (idx + 1)) == '*' && (str charAt (idx + 2)) == '*')
+        skipWhitespace(str, idx + 3)
+      else idx
+    }
+
+  /** Skips to next occurrence of `\n` or to the position after the `/``**` sequence following index `start`.
+   */
+  def skipToEol(str: String, start: Int): Int =
+    if (start + 2 < str.length && (str charAt start) == '/' && (str charAt (start + 1)) == '*' && (str charAt (start + 2)) == '*') start + 3
+    else if (start < str.length && (str charAt start) != '\n') skipToEol(str, start + 1)
+    else start
+
+  /** Returns first index following `start` and starting a line (i.e. after skipLineLead) or starting the comment
+   *  which satisfies predicate `p`.
+   */
+  def findNext(str: String, start: Int)(p: Int => Boolean): Int = {
+    val idx = skipLineLead(str, skipToEol(str, start))
+    if (idx < str.length && !p(idx)) findNext(str, idx)(p)
+    else idx
+  }
+
+  /** Return first index following `start` and starting a line (i.e. after skipLineLead)
+   *  which satisfies predicate `p`.
+   */
+  def findAll(str: String, start: Int)(p: Int => Boolean): List[Int] = {
+    val idx = findNext(str, start)(p)
+    if (idx == str.length) List()
+    else idx :: findAll(str, idx)(p)
+  }
+
+  /** Produces a string index, which is a list of `sections`, i.e
+   *  pairs of start/end positions of all tagged sections in the string.
+   *  Every section starts with an at sign and extends to the next at sign,
+   *  or to the end of the comment string, but excluding the final two
+   *  characters which terminate the comment.
+   *
+   *  Also take usecases into account - they need to expand until the next
+   *  usecase or the end of the string, as they might include other sections
+   *  of their own
+   */
+  def tagIndex(str: String, p: Int => Boolean = (idx => true)): List[(Int, Int)] = {
+    var indices = findAll(str, 0) (idx => str(idx) == '@' && p(idx))
+    indices = mergeUsecaseSections(str, indices)
+    indices = mergeInheritdocSections(str, indices)
+
+    indices match {
+      case List() => List()
+      case idxs   => idxs zip (idxs.tail ::: List(str.length - 2))
+    }
+  }
+
+  /**
+   * Merge sections following an usecase into the usecase comment, so they
+   * can override the parent symbol's sections
+   */
+  def mergeUsecaseSections(str: String, idxs: List[Int]): List[Int] = {
+    idxs.indexWhere(str.startsWith("@usecase", _)) match {
+      case firstUCIndex if firstUCIndex != -1 =>
+        val commentSections = idxs.take(firstUCIndex)
+        val usecaseSections = idxs.drop(firstUCIndex).filter(str.startsWith("@usecase", _))
+        commentSections ::: usecaseSections
+      case _ =>
+        idxs
+    }
+  }
+
+  /**
+   * Merge the inheritdoc sections, as they never make sense on their own
+   */
+  def mergeInheritdocSections(str: String, idxs: List[Int]): List[Int] =
+    idxs.filterNot(str.startsWith("@inheritdoc", _))
+
+  /** Does interval `iv` start with given `tag`?
+   */
+  def startsWithTag(str: String, section: (Int, Int), tag: String): Boolean =
+    startsWithTag(str, section._1, tag)
+
+  def startsWithTag(str: String, start: Int, tag: String): Boolean =
+    str.startsWith(tag, start) && !isIdentifierPart(str charAt (start + tag.length))
+
+  /** The first start tag of a list of tag intervals,
+   *  or the end of the whole comment string - 2 if list is empty
+   */
+  def startTag(str: String, sections: List[(Int, Int)]) = sections match {
+    case Nil             => str.length - 2
+    case (start, _) :: _ => start
+  }
+
+  /** A map from parameter names to start/end indices describing all parameter
+   *  sections in `str` tagged with `tag`, where `sections` is the index of `str`.
+   */
+  def paramDocs(str: String, tag: String, sections: List[(Int, Int)]): Map[String, (Int, Int)] =
+    Map() ++ {
+      for (section <- sections if startsWithTag(str, section, tag)) yield {
+        val start = skipWhitespace(str, section._1 + tag.length)
+        str.substring(start, skipIdent(str, start)) -> section
+      }
+    }
+
+  /** Optionally start and end index of return section in `str`, or `None`
+   *  if `str` does not have a @group. */
+  def groupDoc(str: String, sections: List[(Int, Int)]): Option[(Int, Int)] =
+    sections find (startsWithTag(str, _, "@group"))
+
+
+  /** Optionally start and end index of return section in `str`, or `None`
+   *  if `str` does not have a @return.
+   */
+  def returnDoc(str: String, sections: List[(Int, Int)]): Option[(Int, Int)] =
+    sections find (startsWithTag(str, _, "@return"))
+
+  /** Extracts variable name from a string, stripping any pair of surrounding braces */
+  def variableName(str: String): String =
+    if (str.length >= 2 && (str charAt 0) == '{' && (str charAt (str.length - 1)) == '}')
+      str.substring(1, str.length - 1)
+    else
+      str
+
+  /** Returns index following variable, or start index if no variable was recognized
+   */
+  def skipVariable(str: String, start: Int): Int = {
+    var idx = start
+    if (idx < str.length && (str charAt idx) == '{') {
+      do idx += 1
+      while (idx < str.length && (str charAt idx) != '}')
+      if (idx < str.length) idx + 1 else start
+    } else {
+      while (idx < str.length && isVarPart(str charAt idx))
+        idx += 1
+      idx
+    }
+  }
+
+  /** A map from the section tag to section parameters */
+  def sectionTagMap(str: String, sections: List[(Int, Int)]): Map[String, (Int, Int)] =
+    Map() ++ {
+      for (section <- sections) yield
+        extractSectionTag(str, section) -> section
+    }
+
+  /** Extract the section tag, treating the section tag as an identifier */
+  def extractSectionTag(str: String, section: (Int, Int)): String =
+    str.substring(section._1, skipTag(str, section._1))
+
+  /** Extract the section parameter */
+  def extractSectionParam(str: String, section: (Int, Int)): String = {
+    val (beg, _) = section
+    assert(str.startsWith("@param", beg) ||
+           str.startsWith("@tparam", beg) ||
+           str.startsWith("@throws", beg))
+
+    val start = skipWhitespace(str, skipTag(str, beg))
+    val finish = skipIdent(str, start)
+
+    str.substring(start, finish)
+  }
+
+  /** Extract the section text, except for the tag and comment newlines */
+  def extractSectionText(str: String, section: (Int, Int)): (Int, Int) = {
+    val (beg, end) = section
+    if (str.startsWith("@param", beg) ||
+        str.startsWith("@tparam", beg) ||
+        str.startsWith("@throws", beg))
+      (skipWhitespace(str, skipIdent(str, skipWhitespace(str, skipTag(str, beg)))), end)
+    else
+      (skipWhitespace(str, skipTag(str, beg)), end)
+  }
+
+  /** Cleanup section text */
+  def cleanupSectionText(str: String) = {
+    var result = str.trim.replaceAll("\n\\s+\\*\\s+", " \n")
+    while (result.endsWith("\n"))
+      result = result.substring(0, str.length - 1)
+    result
+  }
+
+
+  def removeSections(raw: String, xs: String*): String = {
+    val sections = tagIndex(raw)
+
+    val toBeRemoved = for {
+      section <- xs
+      lines = sections filter { startsWithTag(raw, _, section) }
+    } yield lines
+
+    val end = startTag(raw, toBeRemoved.flatten.sortBy(_._1).toList)
+
+    if (end == raw.length - 2) raw else raw.substring(0, end) + "*/"
+  }
+}