Separate `ContextDocstrings` from `Context` and make it pluggable

13 files changed, 143 insertions, 109 deletions

diff --git a/dottydoc/src/dotty/tools/dottydoc/DottyDoc.scala b/dottydoc/src/dotty/tools/dottydoc/DottyDoc.scala
index a80efb165..0dea96134 100644
--- a/dottydoc/src/dotty/tools/dottydoc/DottyDoc.scala
+++ b/dottydoc/src/dotty/tools/dottydoc/DottyDoc.scala
@@ -7,6 +7,7 @@ import core.transform._
 import dotc.config.CompilerCommand
 import dotc.config.Printers.dottydoc
 import dotc.core.Contexts._
+import dotc.core.Comments.ContextDoc
 import dotc.core.Phases.Phase
 import dotc.typer.FrontEnd
 import dotc.{ CompilationUnit, Compiler, Driver, Run }
@@ -57,7 +58,7 @@ abstract class DocDriver extends Driver {
 
     ctx.setSettings(summary.sstate)
     ctx.setSetting(ctx.settings.YkeepComments, true)
-    ctx.setProperty(DocContext, new DocBase)
+    ctx.setProperty(ContextDoc, new ContextDottydoc)
 
     val fileNames = CompilerCommand.checkUsage(summary, sourcesRequired)(ctx)
     (fileNames, ctx)
diff --git a/dottydoc/src/dotty/tools/dottydoc/core/ContextDottydoc.scala b/dottydoc/src/dotty/tools/dottydoc/core/ContextDottydoc.scala
new file mode 100644
index 000000000..c60038836
--- /dev/null
+++ b/dottydoc/src/dotty/tools/dottydoc/core/ContextDottydoc.scala
@@ -0,0 +1,23 @@
+package dotty.tools
+package dottydoc
+package core
+
+import dotc.core.Symbols.Symbol
+import dotc.core.Comments.ContextDocstrings
+import model.Package
+
+class ContextDottydoc extends ContextDocstrings {
+  import scala.collection.mutable
+
+  private[this] val _packages: mutable.Map[String, Package] = mutable.Map.empty
+  def packages: Map[String, Package] = _packages.toMap
+  def packagesMutable: mutable.Map[String, Package] = _packages
+
+  /** Should perhaps factorize this into caches that get flushed */
+  private var _defs: Map[Symbol, Set[Symbol]] = Map.empty
+  def defs(sym: Symbol): Set[Symbol] = _defs.get(sym).getOrElse(Set.empty)
+
+  def addDef(s: Symbol, d: Symbol): Unit = _defs = (_defs + {
+    s -> _defs.get(s).map(xs => xs + d).getOrElse(Set(d))
+  })
+}
diff --git a/dottydoc/src/dotty/tools/dottydoc/core/DocASTPhase.scala b/dottydoc/src/dotty/tools/dottydoc/core/DocASTPhase.scala
index c6e9d6190..806d9d0ae 100644
--- a/dottydoc/src/dotty/tools/dottydoc/core/DocASTPhase.scala
+++ b/dottydoc/src/dotty/tools/dottydoc/core/DocASTPhase.scala
@@ -6,7 +6,8 @@ package core
 import dotc.ast.Trees._
 import dotc.CompilationUnit
 import dotc.config.Printers.dottydoc
-import dotc.core.Contexts.{ Context, DocBase }
+import dotc.core.Contexts.Context
+import dotc.core.Comments.ContextDocstrings
 import dotc.core.Phases.Phase
 import dotc.core.Symbols.{ Symbol, NoSymbol }
 
diff --git a/dottydoc/src/dotty/tools/dottydoc/core/MiniPhaseTransform.scala b/dottydoc/src/dotty/tools/dottydoc/core/MiniPhaseTransform.scala
index 201640e4a..150a4e08f 100644
--- a/dottydoc/src/dotty/tools/dottydoc/core/MiniPhaseTransform.scala
+++ b/dottydoc/src/dotty/tools/dottydoc/core/MiniPhaseTransform.scala
@@ -3,7 +3,8 @@ package dottydoc
 package core
 
 import dotc.CompilationUnit
-import dotc.core.Contexts.{ Context, DocBase }
+import dotc.core.Contexts.Context
+import dotc.core.Comments.ContextDocstrings
 import dotc.core.Phases.Phase
 import model._
 import model.internal._
diff --git a/dottydoc/src/dotty/tools/dottydoc/util/syntax.scala b/dottydoc/src/dotty/tools/dottydoc/util/syntax.scala
index 140b3e761..dd3d21f8d 100644
--- a/dottydoc/src/dotty/tools/dottydoc/util/syntax.scala
+++ b/dottydoc/src/dotty/tools/dottydoc/util/syntax.scala
@@ -2,20 +2,15 @@ package dotty.tools
 package dottydoc
 package util
 
-import dotc.core.Contexts.{ Context, DocBase }
+import dotc.core.Contexts.Context
+import dotc.core.Comments._
 import model.Package
+import core.ContextDottydoc
 
 object syntax {
-  implicit class RichDocContext(val ctx: Context) extends AnyVal {
-    def docbase: DocBase = ctx.getDocbase getOrElse {
+  implicit class ContextWithContextDottydoc(val ctx: Context) extends AnyVal {
+    def docbase: ContextDottydoc = ctx.docCtx.getOrElse {
       throw new IllegalStateException("DocBase must be set before running dottydoc phases")
-    }
-  }
-
-  implicit class RichDocBase(val db: DocBase) {
-    def packages: Map[String, Package] = db.packagesAs[Package].toMap
-
-    def packagesMutable: collection.mutable.Map[String, Package] =
-      db.packagesAs[Package]
+    }.asInstanceOf[ContextDottydoc]
   }
 }
diff --git a/dottydoc/test/BaseTest.scala b/dottydoc/test/BaseTest.scala
index 4dff1d4d1..dff112a7f 100644
--- a/dottydoc/test/BaseTest.scala
+++ b/dottydoc/test/BaseTest.scala
@@ -1,12 +1,12 @@
 package dotty.tools
 package dottydoc
 
-import dotc.core.Contexts
-import Contexts.{ Context, ContextBase, FreshContext, DocContext, DocBase }
+import dotc.core.Contexts.{ Context, ContextBase, FreshContext }
+import dotc.core.Comments.{ ContextDoc, ContextDocstrings }
 import dotc.util.SourceFile
 import dotc.core.Phases.Phase
 import dotc.typer.FrontEnd
-import dottydoc.core.DocASTPhase
+import dottydoc.core.{ DocASTPhase, ContextDottydoc }
 import model.Package
 import dotty.tools.dottydoc.util.syntax._
 
@@ -20,7 +20,7 @@ trait DottyTest {
     ctx.setSetting(ctx.settings.language, List("Scala2"))
     ctx.setSetting(ctx.settings.YnoInline, true)
     ctx.setSetting(ctx.settings.YkeepComments, true)
-    ctx.setProperty(DocContext, new DocBase)
+    ctx.setProperty(ContextDoc, new ContextDottydoc)
     base.initialize()(ctx)
     ctx
   }
diff --git a/src/dotty/tools/dotc/core/Comments.scala b/src/dotty/tools/dotc/core/Comments.scala
index 4f36b3b6b..8c2443c9d 100644
--- a/src/dotty/tools/dotc/core/Comments.scala
+++ b/src/dotty/tools/dotc/core/Comments.scala
@@ -2,17 +2,40 @@ package dotty.tools
 package dotc
 package core
 
-import dotc.ast.{ untpd, tpd }
-import Decorators._
-import Symbols._
-import Contexts.Context
-import Flags.EmptyFlags
-import dotc.util.SourceFile
-import dotc.util.Positions._
-import dotc.util.CommentParsing._
-import dotc.parsing.Parsers.Parser
+import ast.{ untpd, tpd }
+import Decorators._, Symbols._, Contexts._, Flags.EmptyFlags
+import util.SourceFile
+import util.Positions._
+import util.CommentParsing._
+import util.Property.Key
+import parsing.Parsers.Parser
 
 object Comments {
+  val ContextDoc =  new Key[ContextDocstrings]
+
+  /** Decorator for getting docbase out of context */
+  implicit class CommentsContext(val ctx: Context) extends AnyVal {
+    def docCtx: Option[ContextDocstrings] = ctx.property(ContextDoc)
+  }
+
+  /** Context for Docstrings, contains basic functionality for getting
+    * docstrings via `Symbol` and expanding templates
+    */
+  class ContextDocstrings {
+    import scala.collection.mutable
+
+    private[this] val _docstrings: mutable.Map[Symbol, Comment] =
+      mutable.Map.empty
+
+    val templateExpander = new CommentExpander
+
+    def docstrings: Map[Symbol, Comment] = _docstrings.toMap
+
+    def docstring(sym: Symbol): Option[Comment] = _docstrings.get(sym)
+
+    def addDocstring(sym: Symbol, doc: Option[Comment]): Unit =
+      doc.map(d => _docstrings += (sym -> d))
+  }
 
   abstract case class Comment(pos: Position, raw: String)(implicit ctx: Context) { self =>
     def isExpanded: Boolean
@@ -155,7 +178,7 @@ object Comments {
      */
     def cookedDocComment(sym: Symbol, docStr: String = "")(implicit ctx: Context): String = cookedDocComments.getOrElseUpdate(sym, {
       var ownComment =
-        if (docStr.length == 0) ctx.getDocbase.flatMap(_.docstring(sym).map(c => template(c.raw))).getOrElse("")
+        if (docStr.length == 0) ctx.docCtx.flatMap(_.docstring(sym).map(c => template(c.raw))).getOrElse("")
         else template(docStr)
       ownComment = replaceInheritDocToInheritdoc(ownComment)
 
@@ -365,7 +388,7 @@ object Comments {
     def defineVariables(sym: Symbol)(implicit ctx: Context) = {
       val Trim = "(?s)^[\\s&&[^\n\r]]*(.*?)\\s*$".r
 
-      val raw = ctx.getDocbase.flatMap(_.docstring(sym).map(_.raw)).getOrElse("")
+      val raw = ctx.docCtx.flatMap(_.docstring(sym).map(_.raw)).getOrElse("")
       defs(sym) ++= defines(raw).map {
         str => {
           val start = skipWhitespace(str, "@define".length)
@@ -406,7 +429,7 @@ object Comments {
      *  the position of the doc comment of the overridden version is returned instead.
      */
     def docCommentPos(sym: Symbol)(implicit ctx: Context): Position =
-      ctx.getDocbase.flatMap(_.docstring(sym).map(_.pos)).getOrElse(NoPosition)
+      ctx.docCtx.flatMap(_.docstring(sym).map(_.pos)).getOrElse(NoPosition)
 
     /** A version which doesn't consider self types, as a temporary measure:
      *  an infinite loop has broken out between superComment and cookedDocComment
diff --git a/src/dotty/tools/dotc/core/Contexts.scala b/src/dotty/tools/dotc/core/Contexts.scala
index bf171bf60..5c9fdaf88 100644
--- a/src/dotty/tools/dotc/core/Contexts.scala
+++ b/src/dotty/tools/dotc/core/Contexts.scala
@@ -69,9 +69,6 @@ object Contexts {
     /** The context base at the root */
     val base: ContextBase
 
-    /** Documentation base */
-    def getDocbase = property(DocContext)
-
     /** All outer contexts, ending in `base.initialCtx` and then `NoContext` */
     def outersIterator = new Iterator[Context] {
       var current = thiscontext
@@ -578,37 +575,6 @@ object Contexts {
     }
   }
 
-  val DocContext = new Key[DocBase]
-  class DocBase {
-    private[this] val _docstrings: mutable.Map[Symbol, Comment] =
-      mutable.Map.empty
-
-    val templateExpander = new CommentExpander
-
-    def docstrings: Map[Symbol, Comment] = _docstrings.toMap
-
-    def docstring(sym: Symbol): Option[Comment] = _docstrings.get(sym)
-
-    def addDocstring(sym: Symbol, doc: Option[Comment]): Unit =
-      doc.map(d => _docstrings += (sym -> d))
-
-    /*
-     * Dottydoc places instances of `Package` in this map - but we do not want
-     * to depend on `dottydoc` for the compiler, as such this is defined as a
-     * map of `String -> AnyRef`
-     */
-    private[this] val _packages: mutable.Map[String, AnyRef] = mutable.Map.empty
-    def packagesAs[A]: mutable.Map[String, A] = _packages.asInstanceOf[mutable.Map[String, A]]
-
-    /** Should perhaps factorize this into caches that get flushed */
-    private var _defs: Map[Symbol, Set[Symbol]] = Map.empty
-    def defs(sym: Symbol): Set[Symbol] = _defs.get(sym).getOrElse(Set.empty)
-
-    def addDef(s: Symbol, d: Symbol): Unit = _defs = (_defs + {
-      s -> _defs.get(s).map(xs => xs + d).getOrElse(Set(d))
-    })
-  }
-
   /** The essential mutable state of a context base, collected into a common class */
   class ContextState {
     // Symbols state
diff --git a/src/dotty/tools/dotc/core/SymDenotations.scala b/src/dotty/tools/dotc/core/SymDenotations.scala
index e2cb378b5..5a5eacd18 100644
--- a/src/dotty/tools/dotc/core/SymDenotations.scala
+++ b/src/dotty/tools/dotc/core/SymDenotations.scala
@@ -3,7 +3,7 @@ package dotc
 package core
 
 import Periods._, Contexts._, Symbols._, Denotations._, Names._, NameOps._, Annotations._
-import Types._, Flags._, Decorators._, DenotTransformers._, StdNames._, Scopes._
+import Types._, Flags._, Decorators._, DenotTransformers._, StdNames._, Scopes._, Comments._
 import NameOps._
 import Scopes.Scope
 import collection.mutable
@@ -1541,7 +1541,7 @@ object SymDenotations {
 
     /** Enter a symbol in given `scope` without potentially replacing the old copy. */
     def enterNoReplace(sym: Symbol, scope: MutableScope)(implicit ctx: Context): Unit = {
-      def isUsecase = ctx.property(DocContext).isDefined && sym.name.show.takeRight(4) == "$doc"
+      def isUsecase = ctx.docCtx.isDefined && sym.name.show.takeRight(4) == "$doc"
       require(
           (sym.denot.flagsUNSAFE is Private) ||
           !(this is Frozen) ||
diff --git a/src/dotty/tools/dotc/typer/Docstrings.scala b/src/dotty/tools/dotc/typer/Docstrings.scala
new file mode 100644
index 000000000..370844e65
--- /dev/null
+++ b/src/dotty/tools/dotc/typer/Docstrings.scala
@@ -0,0 +1,56 @@
+package dotty.tools
+package dotc
+package typer
+
+import core._
+import Contexts._, Symbols._, Decorators._, Comments._
+import util.Positions._
+import ast.tpd
+
+trait Docstrings { self: Typer =>
+
+  /** The Docstrings typer will handle the expansion of `@define` and
+    * `@inheritdoc` if there is a `DocContext` present as a property in the
+    * supplied `ctx`.
+    *
+    * It will also type any `@usecase` available in function definitions.
+    */
+  def cookComments(syms: List[Symbol], owner: Symbol)(implicit ctx: Context): Unit =
+    ctx.docCtx.foreach { docbase =>
+      val relevantSyms = syms.filter(docbase.docstring(_).isDefined)
+      relevantSyms.foreach { sym =>
+        expandParentDocs(sym)
+        val usecases = docbase.docstring(sym).map(_.usecases).getOrElse(Nil)
+
+        usecases.foreach { usecase =>
+          enterSymbol(createSymbol(usecase.untpdCode))
+
+          typedStats(usecase.untpdCode :: Nil, owner) match {
+            case List(df: tpd.DefDef) => usecase.tpdCode = df
+            case _ => ctx.error("`@usecase` was not a valid definition", usecase.codePos)
+          }
+        }
+      }
+    }
+
+  private def expandParentDocs(sym: Symbol)(implicit ctx: Context): Unit =
+    ctx.docCtx.foreach { docCtx =>
+      docCtx.docstring(sym).foreach { cmt =>
+        def expandDoc(owner: Symbol): Unit = if (!cmt.isExpanded) {
+          val tplExp = docCtx.templateExpander
+          tplExp.defineVariables(sym)
+
+          val newCmt = cmt
+            .expand(tplExp.expandedDocComment(sym, owner, _))
+            .withUsecases
+
+          docCtx.addDocstring(sym, Some(newCmt))
+        }
+
+        if (sym ne NoSymbol) {
+          expandParentDocs(sym.owner)
+          expandDoc(sym.owner)
+        }
+      }
+    }
+}
diff --git a/src/dotty/tools/dotc/typer/Namer.scala b/src/dotty/tools/dotc/typer/Namer.scala
index 938b280d1..4f4278468 100644
--- a/src/dotty/tools/dotc/typer/Namer.scala
+++ b/src/dotty/tools/dotc/typer/Namer.scala
@@ -457,7 +457,7 @@ class Namer { typer: Typer =>
 
   def setDocstring(sym: Symbol, tree: Tree)(implicit ctx: Context) = tree match {
     case t: MemberDef if t.rawComment.isDefined =>
-      ctx.getDocbase.foreach(_.addDocstring(sym, t.rawComment))
+      ctx.docCtx.foreach(_.addDocstring(sym, t.rawComment))
     case _ => ()
   }
 
diff --git a/src/dotty/tools/dotc/typer/Typer.scala b/src/dotty/tools/dotc/typer/Typer.scala
index 3ed6c9228..3e3bb32f5 100644
--- a/src/dotty/tools/dotc/typer/Typer.scala
+++ b/src/dotty/tools/dotc/typer/Typer.scala
@@ -58,7 +58,7 @@ object Typer {
       assert(tree.pos.exists, s"position not set for $tree # ${tree.uniqueId}")
 }
 
-class Typer extends Namer with TypeAssigner with Applications with Implicits with Dynamic with Checking {
+class Typer extends Namer with TypeAssigner with Applications with Implicits with Dynamic with Checking with Docstrings {
 
   import Typer._
   import tpd.{cpy => _, _}
@@ -1247,8 +1247,8 @@ class Typer extends Namer with TypeAssigner with Applications with Implicits wit
     val dummy = localDummy(cls, impl)
     val body1 = typedStats(impl.body, dummy)(inClassContext(self1.symbol))
 
-    if (ctx.property(DocContext).isDefined)
-      typedUsecases(body1.map(_.symbol), self1.symbol)(localContext(cdef, cls).setNewScope)
+    // Expand comments and type usecases
+    cookComments(body1.map(_.symbol), self1.symbol)(localContext(cdef, cls).setNewScope)
 
     checkNoDoubleDefs(cls)
     val impl1 = cpy.Template(impl)(constr1, parents1, self1, body1)
@@ -1538,45 +1538,6 @@ class Typer extends Namer with TypeAssigner with Applications with Implicits wit
     tpd.cpy.DefDef(mdef)(rhs = Inliner.bodyToInline(mdef.symbol)) ::
         Inliner.removeInlineAccessors(mdef.symbol)
 
-  private def typedUsecases(syms: List[Symbol], owner: Symbol)(implicit ctx: Context): Unit =
-    ctx.getDocbase.foreach { docbase =>
-      val relevantSyms = syms.filter(docbase.docstring(_).isDefined)
-      relevantSyms.foreach { sym =>
-        expandParentDocs(sym)
-        val usecases = docbase.docstring(sym).map(_.usecases).getOrElse(Nil)
-
-        usecases.foreach { usecase =>
-          enterSymbol(createSymbol(usecase.untpdCode))
-
-          typedStats(usecase.untpdCode :: Nil, owner) match {
-            case List(df: tpd.DefDef) => usecase.tpdCode = df
-            case _ => ctx.error("`@usecase` was not a valid definition", usecase.codePos)
-          }
-        }
-      }
-    }
-
-  private def expandParentDocs(sym: Symbol)(implicit ctx: Context): Unit =
-    ctx.getDocbase.foreach { docbase =>
-      docbase.docstring(sym).foreach { cmt =>
-        def expandDoc(owner: Symbol): Unit = if (!cmt.isExpanded) {
-          val tplExp = docbase.templateExpander
-          tplExp.defineVariables(sym)
-
-          val newCmt = cmt
-            .expand(tplExp.expandedDocComment(sym, owner, _))
-            .withUsecases
-
-          docbase.addDocstring(sym, Some(newCmt))
-        }
-
-        if (sym ne NoSymbol) {
-          expandParentDocs(sym.owner)
-          expandDoc(sym.owner)
-        }
-      }
-    }
-
   def typedExpr(tree: untpd.Tree, pt: Type = WildcardType)(implicit ctx: Context): Tree =
     typed(tree, pt)(ctx retractMode Mode.PatternOrType)
   def typedType(tree: untpd.Tree, pt: Type = WildcardType)(implicit ctx: Context): Tree = // todo: retract mode between Type and Pattern?
diff --git a/src/dotty/tools/dotc/util/CommentParsing.scala b/src/dotty/tools/dotc/util/CommentParsing.scala
index 077776b5d..cc790d683 100644
--- a/src/dotty/tools/dotc/util/CommentParsing.scala
+++ b/src/dotty/tools/dotc/util/CommentParsing.scala
@@ -5,6 +5,13 @@
  */
 package dotty.tools.dotc.util
 
+/** The comment parsing in `dotc` is used by both the comment cooking and the
+  * dottydoc tool.
+  *
+  * The comment cooking is used to expand comments with `@inheritdoc` and
+  * `@define` annotations. The rest of the comment is untouched and later
+  * handled by dottydoc.
+  */
 object CommentParsing {
   import scala.reflect.internal.Chars._