diff options
author | Felix Mulder <felix.mulder@gmail.com> | 2016-10-06 18:55:42 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2016-10-06 18:55:42 +0200 |
commit | 0426fe7e2cc47ad5841e3580e1686b30f33e6d3a (patch) | |
tree | 6247cc96ff18cdde53c82f09ee0694e792869d27 | |
parent | 10ff9494165210b22eb80e989fc10c3ebf393bae (diff) | |
parent | 9bcf282243bb3d6464d95a13c3ac9723dea329d8 (diff) | |
download | dotty-0426fe7e2cc47ad5841e3580e1686b30f33e6d3a.tar.gz dotty-0426fe7e2cc47ad5841e3580e1686b30f33e6d3a.tar.bz2 dotty-0426fe7e2cc47ad5841e3580e1686b30f33e6d3a.zip |
Merge pull request #1502 from felixmulder/topic/dottydoc-usecases-param
[doc] Usecases with docbase as param
28 files changed, 1037 insertions, 613 deletions
diff --git a/dottydoc/src/dotty/tools/dottydoc/DottyDoc.scala b/dottydoc/src/dotty/tools/dottydoc/DottyDoc.scala index 2d4c7abcf..0dea96134 100644 --- a/dottydoc/src/dotty/tools/dottydoc/DottyDoc.scala +++ b/dottydoc/src/dotty/tools/dottydoc/DottyDoc.scala @@ -1,11 +1,13 @@ package dotty.tools package dottydoc +import dotty.tools.dottydoc.util.syntax._ import core._ import core.transform._ import dotc.config.CompilerCommand import dotc.config.Printers.dottydoc import dotc.core.Contexts._ +import dotc.core.Comments.ContextDoc import dotc.core.Phases.Phase import dotc.typer.FrontEnd import dotc.{ CompilationUnit, Compiler, Driver, Run } @@ -31,7 +33,9 @@ class DocCompiler extends Compiler { List(new DocFrontEnd), List(new DocImplicitsPhase), List(new DocASTPhase), - List(DocMiniTransformations(new LinkReturnTypes, + List(DocMiniTransformations(new UsecasePhase, + new DocstringPhase, + new LinkReturnTypes, new LinkParamListTypes, new LinkImplicitlyAddedTypes, new LinkSuperTypes, @@ -54,6 +58,7 @@ abstract class DocDriver extends Driver { ctx.setSettings(summary.sstate) ctx.setSetting(ctx.settings.YkeepComments, true) + ctx.setProperty(ContextDoc, new ContextDottydoc) val fileNames = CompilerCommand.checkUsage(summary, sourcesRequired)(ctx) (fileNames, ctx) @@ -65,7 +70,7 @@ abstract class DocDriver extends Driver { val (fileNames, ctx) = setup(args, initCtx.fresh) doCompile(newCompiler(ctx), fileNames)(ctx) - ctx.docbase.packages[Package] + ctx.docbase.packages } def compiledDocsJava(args: Array[String]): JMap[String, Package] = diff --git a/dottydoc/src/dotty/tools/dottydoc/core/ContextDottydoc.scala b/dottydoc/src/dotty/tools/dottydoc/core/ContextDottydoc.scala new file mode 100644 index 000000000..c60038836 --- /dev/null +++ b/dottydoc/src/dotty/tools/dottydoc/core/ContextDottydoc.scala @@ -0,0 +1,23 @@ +package dotty.tools +package dottydoc +package core + +import dotc.core.Symbols.Symbol +import dotc.core.Comments.ContextDocstrings +import model.Package + +class ContextDottydoc extends ContextDocstrings { + import scala.collection.mutable + + private[this] val _packages: mutable.Map[String, Package] = mutable.Map.empty + def packages: Map[String, Package] = _packages.toMap + def packagesMutable: mutable.Map[String, Package] = _packages + + /** Should perhaps factorize this into caches that get flushed */ + private var _defs: Map[Symbol, Set[Symbol]] = Map.empty + def defs(sym: Symbol): Set[Symbol] = _defs.get(sym).getOrElse(Set.empty) + + def addDef(s: Symbol, d: Symbol): Unit = _defs = (_defs + { + s -> _defs.get(s).map(xs => xs + d).getOrElse(Set(d)) + }) +} diff --git a/dottydoc/src/dotty/tools/dottydoc/core/DocASTPhase.scala b/dottydoc/src/dotty/tools/dottydoc/core/DocASTPhase.scala index 7744752ce..806d9d0ae 100644 --- a/dottydoc/src/dotty/tools/dottydoc/core/DocASTPhase.scala +++ b/dottydoc/src/dotty/tools/dottydoc/core/DocASTPhase.scala @@ -7,6 +7,7 @@ import dotc.ast.Trees._ import dotc.CompilationUnit import dotc.config.Printers.dottydoc import dotc.core.Contexts.Context +import dotc.core.Comments.ContextDocstrings import dotc.core.Phases.Phase import dotc.core.Symbols.{ Symbol, NoSymbol } @@ -14,29 +15,17 @@ class DocASTPhase extends Phase { import model._ import model.factories._ import model.internal._ - import model.parsers.WikiParser import model.comment.Comment import dotty.tools.dotc.core.Flags import dotty.tools.dotc.ast.tpd._ + import dotty.tools.dottydoc.util.syntax._ import util.traversing._ import util.internal.setters._ def phaseName = "docphase" - private[this] val commentParser = new WikiParser - - /** Saves the commentParser function for later evaluation, for when the AST has been filled */ - def track(symbol: Symbol, ctx: Context, parent: Symbol = NoSymbol)(op: => Entity) = { - val entity = op - - if (entity != NonEntity) - commentParser += (entity, symbol, parent, ctx) - - entity - } - /** Build documentation hierarchy from existing tree */ - def collect(tree: Tree, prev: List[String] = Nil)(implicit ctx: Context): Entity = track(tree.symbol, ctx) { + def collect(tree: Tree, prev: List[String] = Nil)(implicit ctx: Context): Entity = { val implicitConversions = ctx.docbase.defs(tree.symbol) def collectList(xs: List[Tree], ps: List[String]): List[Entity] = @@ -58,28 +47,26 @@ class DocASTPhase extends Phase { val defs = sym.info.bounds.hi.membersBasedOnFlags(Flags.Method, Flags.Synthetic | Flags.Private) .filterNot(_.symbol.owner.name.show == "Any") .map { meth => - track(meth.symbol, ctx, tree.symbol) { - DefImpl( - meth.symbol.name.show, - Nil, - path(meth.symbol), - returnType(meth.info), - typeParams(meth.symbol), - paramLists(meth.info), - implicitlyAddedFrom = Some(returnType(meth.symbol.owner.info)) - ) - } + DefImpl( + meth.symbol, + meth.symbol.name.show, + Nil, + path(meth.symbol), + returnType(meth.info), + typeParams(meth.symbol), + paramLists(meth.info), + implicitlyAddedFrom = Some(returnType(meth.symbol.owner.info)) + ) }.toList val vals = sym.info.fields.filterNot(_.symbol.is(Flags.Private | Flags.Synthetic)).map { value => - track(value.symbol, ctx, tree.symbol) { - ValImpl( - value.symbol.name.show, - Nil, path(value.symbol), - returnType(value.info), - implicitlyAddedFrom = Some(returnType(value.symbol.owner.info)) - ) - } + ValImpl( + value.symbol, + value.symbol.name.show, + Nil, path(value.symbol), + returnType(value.info), + implicitlyAddedFrom = Some(returnType(value.symbol.owner.info)) + ) } defs ++ vals @@ -90,38 +77,38 @@ class DocASTPhase extends Phase { /** package */ case pd @ PackageDef(pid, st) => val newPath = prev :+ pid.name.toString - addEntity(PackageImpl(newPath.mkString("."), collectEntityMembers(st, newPath), newPath)) + addEntity(PackageImpl(pd.symbol, newPath.mkString("."), collectEntityMembers(st, newPath), newPath)) /** trait */ case t @ TypeDef(n, rhs) if t.symbol.is(Flags.Trait) => val name = n.decode.toString val newPath = prev :+ name //TODO: should not `collectMember` from `rhs` - instead: get from symbol, will get inherited members as well - TraitImpl(name, collectMembers(rhs), flags(t), newPath, typeParams(t.symbol), traitParameters(t.symbol), superTypes(t)) + TraitImpl(t.symbol, name, collectMembers(rhs), flags(t), newPath, typeParams(t.symbol), traitParameters(t.symbol), superTypes(t)) /** objects, on the format "Object$" so drop the last letter */ case o @ TypeDef(n, rhs) if o.symbol.is(Flags.Module) => val name = n.decode.toString.dropRight(1) //TODO: should not `collectMember` from `rhs` - instead: get from symbol, will get inherited members as well - ObjectImpl(name, collectMembers(rhs, prev :+ name), flags(o), prev :+ (name + "$"), superTypes(o)) + ObjectImpl(o.symbol, name, collectMembers(rhs, prev :+ name), flags(o), prev :+ (name + "$"), superTypes(o)) /** class / case class */ case c @ TypeDef(n, rhs) if c.symbol.isClass => val name = n.decode.toString val newPath = prev :+ name //TODO: should not `collectMember` from `rhs` - instead: get from symbol, will get inherited members as well - (name, collectMembers(rhs), flags(c), newPath, typeParams(c.symbol), constructors(c.symbol), superTypes(c), None) match { + (c.symbol, name, collectMembers(rhs), flags(c), newPath, typeParams(c.symbol), constructors(c.symbol), superTypes(c), None) match { case x if c.symbol.is(Flags.CaseClass) => CaseClassImpl.tupled(x) case x => ClassImpl.tupled(x) } /** def */ case d: DefDef => - DefImpl(d.name.decode.toString, flags(d), path(d.symbol), returnType(d.tpt.tpe), typeParams(d.symbol), paramLists(d.symbol.info)) + DefImpl(d.symbol, d.name.decode.toString, flags(d), path(d.symbol), returnType(d.tpt.tpe), typeParams(d.symbol), paramLists(d.symbol.info)) /** val */ case v: ValDef if !v.symbol.is(Flags.ModuleVal) => - ValImpl(v.name.decode.toString, flags(v), path(v.symbol), returnType(v.tpt.tpe)) + ValImpl(v.symbol, v.name.decode.toString, flags(v), path(v.symbol), returnType(v.tpt.tpe)) case x => { //dottydoc.println(s"Found unwanted entity: $x (${x.pos},\n${x.show}") @@ -175,15 +162,8 @@ class DocASTPhase extends Phase { child <- parent.children } setParent(child, to = parent) - // (3) Create documentation template from docstrings, with internal links - println("Generating documentation, this might take a while...") - commentParser.parse(packages) - - // (4) Clear caches - commentParser.clear() - - // (5) Update Doc AST in ctx.base - for (kv <- packages) ctx.docbase.packages += kv + // (3) Update Doc AST in ctx.base + for (kv <- packages) ctx.docbase.packagesMutable += kv // Return super's result compUnits diff --git a/dottydoc/src/dotty/tools/dottydoc/core/DocImplicitsPhase.scala b/dottydoc/src/dotty/tools/dottydoc/core/DocImplicitsPhase.scala index f322d7a5a..6577f0cb7 100644 --- a/dottydoc/src/dotty/tools/dottydoc/core/DocImplicitsPhase.scala +++ b/dottydoc/src/dotty/tools/dottydoc/core/DocImplicitsPhase.scala @@ -5,6 +5,7 @@ package core import dotty.tools.dotc.transform.TreeTransforms.{ MiniPhaseTransform, TransformerInfo } import dotty.tools.dotc.core.Flags import dotc.core.Contexts.Context +import util.syntax._ class DocImplicitsPhase extends MiniPhaseTransform { thisTransformer => import dotty.tools.dotc.ast.tpd._ diff --git a/dottydoc/src/dotty/tools/dottydoc/core/DocstringPhase.scala b/dottydoc/src/dotty/tools/dottydoc/core/DocstringPhase.scala new file mode 100644 index 000000000..cff614528 --- /dev/null +++ b/dottydoc/src/dotty/tools/dottydoc/core/DocstringPhase.scala @@ -0,0 +1,47 @@ +package dotty.tools +package dottydoc +package core + +import dotc.core.Contexts.Context +import transform.DocMiniPhase +import model._ +import model.internal._ +import model.comment._ +import BodyParsers._ +import util.syntax._ + +class DocstringPhase extends DocMiniPhase with CommentParser with CommentCleaner { + private def parsedComment[E <: Entity](ent: E)(implicit ctx: Context): Option[Comment] = + ctx.docbase.docstring(ent.symbol).map { cmt => + parse(ent, ctx.docbase.packages, clean(cmt.raw), cmt.raw, cmt.pos) + .toComment(_.toHtml(ent)) + } + + override def transformPackage(implicit ctx: Context) = { case ent: PackageImpl => + ent.copy(comment = parsedComment(ent)) + } + + override def transformClass(implicit ctx: Context) = { case ent: ClassImpl => + ent.copy(comment = parsedComment(ent)) + } + + override def transformCaseClass(implicit ctx: Context) = { case ent: CaseClassImpl => + ent.copy(comment = parsedComment(ent)) + } + + override def transformTrait(implicit ctx: Context) = { case ent: TraitImpl => + ent.copy(comment = parsedComment(ent)) + } + + override def transformObject(implicit ctx: Context) = { case ent: ObjectImpl => + ent.copy(comment = parsedComment(ent)) + } + + override def transformDef(implicit ctx: Context) = { case ent: DefImpl => + ent.copy(comment = parsedComment(ent)) + } + + override def transformVal(implicit ctx: Context) = { case ent: ValImpl => + ent.copy(comment = parsedComment(ent)) + } +} diff --git a/dottydoc/src/dotty/tools/dottydoc/core/MiniPhaseTransform.scala b/dottydoc/src/dotty/tools/dottydoc/core/MiniPhaseTransform.scala index 2690ac7b7..150a4e08f 100644 --- a/dottydoc/src/dotty/tools/dottydoc/core/MiniPhaseTransform.scala +++ b/dottydoc/src/dotty/tools/dottydoc/core/MiniPhaseTransform.scala @@ -4,9 +4,11 @@ package core import dotc.CompilationUnit import dotc.core.Contexts.Context +import dotc.core.Comments.ContextDocstrings import dotc.core.Phases.Phase import model._ import model.internal._ +import util.syntax._ object transform { /** @@ -43,9 +45,9 @@ object transform { override def runOn(units: List[CompilationUnit])(implicit ctx: Context): List[CompilationUnit] = { for { rootName <- rootPackages - pack = ctx.docbase.packages[Package](rootName) + pack = ctx.docbase.packages(rootName) transformed = performPackageTransform(pack) - } yield ctx.docbase.packages(rootName) = transformed + } yield ctx.docbase.packagesMutable(rootName) = transformed super.runOn(units) } @@ -77,6 +79,7 @@ object transform { def traverse(ent: Entity): Entity = ent match { case p: Package => transformEntity(p, _.packageTransformation) { p => val newPackage = PackageImpl( + p.symbol, p.name, p.members.map(traverse), p.path, @@ -84,12 +87,13 @@ object transform { ) // Update reference in context to newPackage - ctx.docbase.packages[Package] += (newPackage.path.mkString(".") -> newPackage) + ctx.docbase.packagesMutable += (newPackage.path.mkString(".") -> newPackage) newPackage } case c: Class => transformEntity(c, _.classTransformation) { cls => ClassImpl( + cls.symbol, cls.name, cls.members.map(traverse), cls.modifiers, @@ -102,6 +106,7 @@ object transform { } case cc: CaseClass => transformEntity(cc, _.caseClassTransformation) { cc => CaseClassImpl( + cc.symbol, cc.name, cc.members.map(traverse), cc.modifiers, @@ -114,6 +119,7 @@ object transform { } case trt: Trait => transformEntity(trt, _.traitTransformation) { trt => TraitImpl( + trt.symbol, trt.name, trt.members.map(traverse), trt.modifiers, @@ -126,6 +132,7 @@ object transform { } case obj: Object => transformEntity(obj, _.objectTransformation) { obj => ObjectImpl( + obj.symbol, obj.name, obj.members.map(traverse), obj.modifiers, @@ -136,6 +143,7 @@ object transform { } case df: Def => transformEntity(df, _.defTransformation) { df => DefImpl( + df.symbol, df.name, df.modifiers, df.path, @@ -148,6 +156,7 @@ object transform { } case vl: Val => transformEntity(vl, _.valTransformation) { vl => ValImpl( + vl.symbol, vl.name, vl.modifiers, vl.path, diff --git a/dottydoc/src/dotty/tools/dottydoc/core/TypeLinkingPhases.scala b/dottydoc/src/dotty/tools/dottydoc/core/TypeLinkingPhases.scala index ae07effa9..1aecca9e1 100644 --- a/dottydoc/src/dotty/tools/dottydoc/core/TypeLinkingPhases.scala +++ b/dottydoc/src/dotty/tools/dottydoc/core/TypeLinkingPhases.scala @@ -14,15 +14,16 @@ import BodyParsers._ import util.MemberLookup import util.traversing._ import util.internal.setters._ +import util.syntax._ class LinkReturnTypes extends DocMiniPhase with TypeLinker { override def transformDef(implicit ctx: Context) = { case df: DefImpl => - val returnValue = linkReference(df, df.returnValue, ctx.docbase.packages[Package].toMap) + val returnValue = linkReference(df, df.returnValue, ctx.docbase.packages) df.copy(returnValue = returnValue) } override def transformVal(implicit ctx: Context) = { case vl: ValImpl => - val returnValue = linkReference(vl, vl.returnValue, ctx.docbase.packages[Package].toMap) + val returnValue = linkReference(vl, vl.returnValue, ctx.docbase.packages) vl.copy(returnValue = returnValue) } } @@ -31,7 +32,7 @@ class LinkParamListTypes extends DocMiniPhase with TypeLinker { override def transformDef(implicit ctx: Context) = { case df: DefImpl => val newParamLists = for { ParamListImpl(list, isImplicit) <- df.paramLists - newList = list.map(linkReference(df, _, ctx.docbase.packages[Package].toMap)) + newList = list.map(linkReference(df, _, ctx.docbase.packages)) } yield ParamListImpl(newList.asInstanceOf[List[NamedReference]], isImplicit) df.copy(paramLists = newParamLists) @@ -42,7 +43,7 @@ class LinkSuperTypes extends DocMiniPhase with TypeLinker { def linkSuperTypes(ent: Entity with SuperTypes)(implicit ctx: Context): List[MaterializableLink] = ent.superTypes.collect { case UnsetLink(title, query) => - val packages = ctx.docbase.packages[Package].toMap + val packages = ctx.docbase.packages val entityLink = makeEntityLink(ent, packages, Text(title), NoPosition, query).link handleEntityLink(title, entityLink, ent) } @@ -67,13 +68,13 @@ class LinkSuperTypes extends DocMiniPhase with TypeLinker { class LinkImplicitlyAddedTypes extends DocMiniPhase with TypeLinker { override def transformDef(implicit ctx: Context) = { case df: DefImpl if df.implicitlyAddedFrom.isDefined => - val implicitlyAddedFrom = linkReference(df, df.implicitlyAddedFrom.get, ctx.docbase.packages[Package].toMap) + val implicitlyAddedFrom = linkReference(df, df.implicitlyAddedFrom.get, ctx.docbase.packages) df.copy(implicitlyAddedFrom = Some(implicitlyAddedFrom)) } override def transformVal(implicit ctx: Context) = { case vl: ValImpl if vl.implicitlyAddedFrom.isDefined => - val implicitlyAddedFrom = linkReference(vl, vl.implicitlyAddedFrom.get, ctx.docbase.packages[Package].toMap) + val implicitlyAddedFrom = linkReference(vl, vl.implicitlyAddedFrom.get, ctx.docbase.packages) vl.copy(implicitlyAddedFrom = Some(implicitlyAddedFrom)) } } diff --git a/dottydoc/src/dotty/tools/dottydoc/core/UsecasePhase.scala b/dottydoc/src/dotty/tools/dottydoc/core/UsecasePhase.scala new file mode 100644 index 000000000..47376eb26 --- /dev/null +++ b/dottydoc/src/dotty/tools/dottydoc/core/UsecasePhase.scala @@ -0,0 +1,31 @@ +package dotty.tools +package dottydoc +package core + +import dotc.core.Contexts.Context +import dotc.ast.tpd + +import transform.DocMiniPhase +import model.internal._ +import model.factories._ +import dotty.tools.dotc.core.Symbols.Symbol +import util.syntax._ + +class UsecasePhase extends DocMiniPhase { + private def defdefToDef(d: tpd.DefDef, sym: Symbol)(implicit ctx: Context) = { + val name = d.name.show.split("\\$").head // UseCase defs get $pos appended to their names + DefImpl( + sym, + name, + flags(d), + path(d.symbol).init :+ name, + returnType(d.tpt.tpe), + typeParams(d.symbol), + paramLists(d.symbol.info) + ) + } + + override def transformDef(implicit ctx: Context) = { case df: DefImpl => + ctx.docbase.docstring(df.symbol).flatMap(_.usecases.headOption.map(_.tpdCode)).map(defdefToDef(_, df.symbol)).getOrElse(df) + } +} diff --git a/dottydoc/src/dotty/tools/dottydoc/model/comment/CommentExpander.scala b/dottydoc/src/dotty/tools/dottydoc/model/comment/CommentExpander.scala deleted file mode 100644 index 32a0d8128..000000000 --- a/dottydoc/src/dotty/tools/dottydoc/model/comment/CommentExpander.scala +++ /dev/null @@ -1,344 +0,0 @@ -/* - * Port of DocComment.scala from nsc - * @author Martin Odersky - * @author Felix Mulder - */ - -package dotty.tools -package dottydoc -package model -package comment - -import dotc.config.Printers.dottydoc -import dotc.core.Contexts.Context -import dotc.core.Symbols._ -import dotc.core.Flags -import dotc.util.Positions._ - -import scala.collection.mutable - -trait CommentExpander { - import CommentUtils._ - - def expand(sym: Symbol, site: Symbol)(implicit ctx: Context): String = { - val parent = if (site != NoSymbol) site else sym - defineVariables(parent) - expandedDocComment(sym, parent) - } - - /** The cooked doc comment of symbol `sym` after variable expansion, or "" if missing. - * - * @param sym The symbol for which doc comment is returned - * @param site The class for which doc comments are generated - * @throws ExpansionLimitExceeded when more than 10 successive expansions - * of the same string are done, which is - * interpreted as a recursive variable definition. - */ - def expandedDocComment(sym: Symbol, site: Symbol, docStr: String = "")(implicit ctx: Context): String = { - // when parsing a top level class or module, use the (module-)class itself to look up variable definitions - val parent = if ((sym.is(Flags.Module) || sym.isClass) && site.is(Flags.Package)) sym - else site - expandVariables(cookedDocComment(sym, docStr), sym, parent) - } - - private def template(raw: String): String = { - val sections = tagIndex(raw) - - val defines = sections filter { startsWithTag(raw, _, "@define") } - val usecases = sections filter { startsWithTag(raw, _, "@usecase") } - - val end = startTag(raw, (defines ::: usecases).sortBy(_._1)) - - if (end == raw.length - 2) raw else raw.substring(0, end) + "*/" - } - - def defines(raw: String): List[String] = { - val sections = tagIndex(raw) - val defines = sections filter { startsWithTag(raw, _, "@define") } - val usecases = sections filter { startsWithTag(raw, _, "@usecase") } - val end = startTag(raw, (defines ::: usecases).sortBy(_._1)) - - defines map { case (start, end) => raw.substring(start, end) } - } - - private def replaceInheritDocToInheritdoc(docStr: String): String = - docStr.replaceAll("""\{@inheritDoc\p{Zs}*\}""", "@inheritdoc") - - /** The cooked doc comment of an overridden symbol */ - protected def superComment(sym: Symbol)(implicit ctx: Context): Option[String] = - allInheritedOverriddenSymbols(sym).iterator map (x => cookedDocComment(x)) find (_ != "") - - private val cookedDocComments = mutable.HashMap[Symbol, String]() - - /** The raw doc comment of symbol `sym`, minus usecase and define sections, augmented by - * missing sections of an inherited doc comment. - * If a symbol does not have a doc comment but some overridden version of it does, - * the doc comment of the overridden version is copied instead. - */ - def cookedDocComment(sym: Symbol, docStr: String = "")(implicit ctx: Context): String = cookedDocComments.getOrElseUpdate(sym, { - var ownComment = - if (docStr.length == 0) ctx.docbase.docstring(sym).map(c => template(c.chrs)).getOrElse("") - else template(docStr) - ownComment = replaceInheritDocToInheritdoc(ownComment) - - superComment(sym) match { - case None => - // SI-8210 - The warning would be false negative when this symbol is a setter - if (ownComment.indexOf("@inheritdoc") != -1 && ! sym.isSetter) - dottydoc.println(s"${sym.pos}: the comment for ${sym} contains @inheritdoc, but no parent comment is available to inherit from.") - ownComment.replaceAllLiterally("@inheritdoc", "<invalid inheritdoc annotation>") - case Some(sc) => - if (ownComment == "") sc - else expandInheritdoc(sc, merge(sc, ownComment, sym), sym) - } - }) - - private def isMovable(str: String, sec: (Int, Int)): Boolean = - startsWithTag(str, sec, "@param") || - startsWithTag(str, sec, "@tparam") || - startsWithTag(str, sec, "@return") - - def merge(src: String, dst: String, sym: Symbol, copyFirstPara: Boolean = false): String = { - val srcSections = tagIndex(src) - val dstSections = tagIndex(dst) - val srcParams = paramDocs(src, "@param", srcSections) - val dstParams = paramDocs(dst, "@param", dstSections) - val srcTParams = paramDocs(src, "@tparam", srcSections) - val dstTParams = paramDocs(dst, "@tparam", dstSections) - val out = new StringBuilder - var copied = 0 - var tocopy = startTag(dst, dstSections dropWhile (!isMovable(dst, _))) - - if (copyFirstPara) { - val eop = // end of comment body (first para), which is delimited by blank line, or tag, or end of comment - (findNext(src, 0)(src.charAt(_) == '\n')) min startTag(src, srcSections) - out append src.substring(0, eop).trim - copied = 3 - tocopy = 3 - } - - def mergeSection(srcSec: Option[(Int, Int)], dstSec: Option[(Int, Int)]) = dstSec match { - case Some((start, end)) => - if (end > tocopy) tocopy = end - case None => - srcSec match { - case Some((start1, end1)) => { - out append dst.substring(copied, tocopy).trim - out append "\n" - copied = tocopy - out append src.substring(start1, end1).trim - } - case None => - } - } - - //TODO: enable this once you know how to get `sym.paramss` - /* - for (params <- sym.paramss; param <- params) - mergeSection(srcParams get param.name.toString, dstParams get param.name.toString) - for (tparam <- sym.typeParams) - mergeSection(srcTParams get tparam.name.toString, dstTParams get tparam.name.toString) - - mergeSection(returnDoc(src, srcSections), returnDoc(dst, dstSections)) - mergeSection(groupDoc(src, srcSections), groupDoc(dst, dstSections)) - */ - - if (out.length == 0) dst - else { - out append dst.substring(copied) - out.toString - } - } - - /** - * Expand inheritdoc tags - * - for the main comment we transform the inheritdoc into the super variable, - * and the variable expansion can expand it further - * - for the param, tparam and throws sections we must replace comments on the spot - * - * This is done separately, for two reasons: - * 1. It takes longer to run compared to merge - * 2. The inheritdoc annotation should not be used very often, as building the comment from pieces severely - * impacts performance - * - * @param parent The source (or parent) comment - * @param child The child (overriding member or usecase) comment - * @param sym The child symbol - * @return The child comment with the inheritdoc sections expanded - */ - def expandInheritdoc(parent: String, child: String, sym: Symbol): String = - if (child.indexOf("@inheritdoc") == -1) - child - else { - val parentSections = tagIndex(parent) - val childSections = tagIndex(child) - val parentTagMap = sectionTagMap(parent, parentSections) - val parentNamedParams = Map() + - ("@param" -> paramDocs(parent, "@param", parentSections)) + - ("@tparam" -> paramDocs(parent, "@tparam", parentSections)) + - ("@throws" -> paramDocs(parent, "@throws", parentSections)) - - val out = new StringBuilder - - def replaceInheritdoc(childSection: String, parentSection: => String) = - if (childSection.indexOf("@inheritdoc") == -1) - childSection - else - childSection.replaceAllLiterally("@inheritdoc", parentSection) - - def getParentSection(section: (Int, Int)): String = { - - def getSectionHeader = extractSectionTag(child, section) match { - case param@("@param"|"@tparam"|"@throws") => param + " " + extractSectionParam(child, section) - case other => other - } - - def sectionString(param: String, paramMap: Map[String, (Int, Int)]): String = - paramMap.get(param) match { - case Some(section) => - // Cleanup the section tag and parameter - val sectionTextBounds = extractSectionText(parent, section) - cleanupSectionText(parent.substring(sectionTextBounds._1, sectionTextBounds._2)) - case None => - dottydoc.println(s"""${sym.pos}: the """" + getSectionHeader + "\" annotation of the " + sym + - " comment contains @inheritdoc, but the corresponding section in the parent is not defined.") - "<invalid inheritdoc annotation>" - } - - child.substring(section._1, section._1 + 7) match { - case param@("@param "|"@tparam"|"@throws") => - sectionString(extractSectionParam(child, section), parentNamedParams(param.trim)) - case _ => - sectionString(extractSectionTag(child, section), parentTagMap) - } - } - - def mainComment(str: String, sections: List[(Int, Int)]): String = - if (str.trim.length > 3) - str.trim.substring(3, startTag(str, sections)) - else - "" - - // Append main comment - out.append("/**") - out.append(replaceInheritdoc(mainComment(child, childSections), mainComment(parent, parentSections))) - - // Append sections - for (section <- childSections) - out.append(replaceInheritdoc(child.substring(section._1, section._2), getParentSection(section))) - - out.append("*/") - out.toString - } - - protected def expandVariables(initialStr: String, sym: Symbol, site: Symbol)(implicit ctx: Context): String = { - val expandLimit = 10 - - def expandInternal(str: String, depth: Int): String = { - if (depth >= expandLimit) - throw new ExpansionLimitExceeded(str) - - val out = new StringBuilder - var copied, idx = 0 - // excluding variables written as \$foo so we can use them when - // necessary to document things like Symbol#decode - def isEscaped = idx > 0 && str.charAt(idx - 1) == '\\' - while (idx < str.length) { - if ((str charAt idx) != '$' || isEscaped) - idx += 1 - else { - val vstart = idx - idx = skipVariable(str, idx + 1) - def replaceWith(repl: String) { - out append str.substring(copied, vstart) - out append repl - copied = idx - } - variableName(str.substring(vstart + 1, idx)) match { - case "super" => - superComment(sym) foreach { sc => - val superSections = tagIndex(sc) - replaceWith(sc.substring(3, startTag(sc, superSections))) - for (sec @ (start, end) <- superSections) - if (!isMovable(sc, sec)) out append sc.substring(start, end) - } - case "" => idx += 1 - case vname => - lookupVariable(vname, site) match { - case Some(replacement) => replaceWith(replacement) - case None => - dottydoc.println(s"Variable $vname undefined in comment for $sym in $site") - } - } - } - } - if (out.length == 0) str - else { - out append str.substring(copied) - expandInternal(out.toString, depth + 1) - } - } - - // We suppressed expanding \$ throughout the recursion, and now we - // need to replace \$ with $ so it looks as intended. - expandInternal(initialStr, 0).replaceAllLiterally("""\$""", "$") - } - - def defineVariables(sym: Symbol)(implicit ctx: Context) = { - val Trim = "(?s)^[\\s&&[^\n\r]]*(.*?)\\s*$".r - - val raw = ctx.docbase.docstring(sym).map(_.chrs).getOrElse("") - defs(sym) ++= defines(raw).map { - str => { - val start = skipWhitespace(str, "@define".length) - val (key, value) = str.splitAt(skipVariable(str, start)) - key.drop(start) -> value - } - } map { - case (key, Trim(value)) => - variableName(key) -> value.replaceAll("\\s+\\*+$", "") - } - } - - /** Maps symbols to the variable -> replacement maps that are defined - * in their doc comments - */ - private val defs = mutable.HashMap[Symbol, Map[String, String]]() withDefaultValue Map() - - /** Lookup definition of variable. - * - * @param vble The variable for which a definition is searched - * @param site The class for which doc comments are generated - */ - def lookupVariable(vble: String, site: Symbol)(implicit ctx: Context): Option[String] = site match { - case NoSymbol => None - case _ => - val searchList = - if (site.flags.is(Flags.Module)) site :: site.info.baseClasses - else site.info.baseClasses - - searchList collectFirst { case x if defs(x) contains vble => defs(x)(vble) } match { - case Some(str) if str startsWith "$" => lookupVariable(str.tail, site) - case res => res orElse lookupVariable(vble, site.owner) - } - } - - /** The position of the raw doc comment of symbol `sym`, or NoPosition if missing - * If a symbol does not have a doc comment but some overridden version of it does, - * the position of the doc comment of the overridden version is returned instead. - */ - def docCommentPos(sym: Symbol)(implicit ctx: Context): Position = - ctx.docbase.docstring(sym).map(_.pos).getOrElse(NoPosition) - - /** A version which doesn't consider self types, as a temporary measure: - * an infinite loop has broken out between superComment and cookedDocComment - * since r23926. - */ - private def allInheritedOverriddenSymbols(sym: Symbol)(implicit ctx: Context): List[Symbol] = { - if (!sym.owner.isClass) Nil - else sym.allOverriddenSymbols.toList.filter(_ != NoSymbol) //TODO: could also be `sym.owner.allOverrid..` - //else sym.owner.ancestors map (sym overriddenSymbol _) filter (_ != NoSymbol) - } - - class ExpansionLimitExceeded(str: String) extends Exception -} diff --git a/dottydoc/src/dotty/tools/dottydoc/model/entities.scala b/dottydoc/src/dotty/tools/dottydoc/model/entities.scala index 76792070c..52379f9ad 100644 --- a/dottydoc/src/dotty/tools/dottydoc/model/entities.scala +++ b/dottydoc/src/dotty/tools/dottydoc/model/entities.scala @@ -3,8 +3,11 @@ package model import comment._ import references._ +import dotty.tools.dotc.core.Symbols.{ Symbol, NoSymbol } trait Entity { + def symbol: Symbol + def name: String /** Path from root, i.e. `scala.Option$` */ @@ -103,6 +106,7 @@ trait Var extends Entity with Modifiers with ReturnValue { trait NonEntity extends Entity { val name = "" + val symbol = NoSymbol val comment = None val path = Nil val kind = "" diff --git a/dottydoc/src/dotty/tools/dottydoc/model/internal.scala b/dottydoc/src/dotty/tools/dottydoc/model/internal.scala index 6afb1ec9b..09f642d0b 100644 --- a/dottydoc/src/dotty/tools/dottydoc/model/internal.scala +++ b/dottydoc/src/dotty/tools/dottydoc/model/internal.scala @@ -3,6 +3,7 @@ package model import comment.Comment import references._ +import dotty.tools.dotc.core.Symbols.Symbol object internal { @@ -11,6 +12,7 @@ object internal { } final case class PackageImpl( + symbol: Symbol, name: String, var members: List[Entity], path: List[String], @@ -21,6 +23,7 @@ object internal { } final case class ClassImpl( + symbol: Symbol, name: String, members: List[Entity], modifiers: List[String], @@ -32,6 +35,7 @@ object internal { ) extends Class with Impl final case class CaseClassImpl( + symbol: Symbol, name: String, members: List[Entity], modifiers: List[String], @@ -43,6 +47,7 @@ object internal { ) extends CaseClass with Impl final case class TraitImpl( + symbol: Symbol, name: String, members: List[Entity], modifiers: List[String], @@ -54,6 +59,7 @@ object internal { ) extends Trait with Impl final case class ObjectImpl( + symbol: Symbol, name: String, members: List[Entity], modifiers: List[String], @@ -63,6 +69,7 @@ object internal { ) extends Object with Impl final case class DefImpl( + symbol: Symbol, name: String, modifiers: List[String], path: List[String], @@ -74,6 +81,7 @@ object internal { ) extends Def with Impl final case class ValImpl( + symbol: Symbol, name: String, modifiers: List[String], path: List[String], diff --git a/dottydoc/src/dotty/tools/dottydoc/model/parsers.scala b/dottydoc/src/dotty/tools/dottydoc/model/parsers.scala deleted file mode 100644 index fa54163e5..000000000 --- a/dottydoc/src/dotty/tools/dottydoc/model/parsers.scala +++ /dev/null @@ -1,98 +0,0 @@ -package dotty.tools -package dottydoc -package model - -import dotc.core.Symbols.Symbol -import dotc.core.Contexts.Context -import dotc.util.Positions.NoPosition - -object parsers { - import comment._ - import BodyParsers._ - import model.internal._ - import util.MemberLookup - import util.traversing._ - import util.internal.setters._ - - class WikiParser extends CommentCleaner with CommentParser with CommentExpander { - private[this] var commentCache: Map[String, (Entity, Map[String, Package]) => Option[Comment]] = Map.empty - - /** Parses comment and returns the path to the entity with an optional comment - * - * The idea here is to use this fact to create `Future[Seq[(String, Option[Comment]]]` - * which can then be awaited near the end of the run - before the pickling. - */ - def parseHtml(sym: Symbol, parent: Symbol, entity: Entity, packages: Map[String, Package])(implicit ctx: Context): (String, Option[Comment]) = { - val cmt = ctx.docbase.docstring(sym).map { d => - val expanded = expand(sym, parent) - parse(entity, packages, clean(expanded), expanded, d.pos).toComment(_.toHtml(entity)) - } - - (entity.path.mkString("."), cmt) - } - - - def add(entity: Entity, symbol: Symbol, parent: Symbol, ctx: Context): Unit = { - val commentParser = { (entity: Entity, packs: Map[String, Package]) => - parseHtml(symbol, parent, entity, packs)(ctx)._2 - } - - /** TODO: this if statement searches for doc comments in parent - * definitions if one is not defined for the current symbol. - * - * It might be a good idea to factor this out of the WikiParser - since - * it mutates the state of docbase sort of silently. - */ - implicit val implCtx = ctx - if (!ctx.docbase.docstring(symbol).isDefined) { - val parentCmt = - symbol.extendedOverriddenSymbols - .find(ctx.docbase.docstring(_).isDefined) - .flatMap(p => ctx.docbase.docstring(p)) - - ctx.docbase.addDocstring(symbol, parentCmt) - } - - - val path = entity.path.mkString(".") - if (!commentCache.contains(path) || ctx.docbase.docstring(symbol).isDefined) - commentCache = commentCache + (path -> commentParser) - } - - def +=(entity: Entity, symbol: Symbol, parent: Symbol, ctx: Context) = add(entity, symbol, parent, ctx) - - def size: Int = commentCache.size - - private def parse(entity: Entity, packs: Map[String, Package]): Option[Comment] = - commentCache(entity.path.mkString("."))(entity, packs) - - def parse(packs: Map[String, Package]): Unit = { - def rootPackages: List[String] = { - var currentDepth = Int.MaxValue - var packages: List[String] = Nil - - for (key <- packs.keys) { - val keyDepth = key.split("\\.").length - packages = - if (keyDepth < currentDepth) { - currentDepth = keyDepth - key :: Nil - } else if (keyDepth == currentDepth) { - key :: packages - } else packages - } - - packages - } - - for (pack <- rootPackages) { - mutateEntities(packs(pack)) { e => - val comment = parse(e, packs) - setComment(e, to = comment) - } - } - } - - def clear(): Unit = commentCache = Map.empty - } -} diff --git a/dottydoc/src/dotty/tools/dottydoc/util/syntax.scala b/dottydoc/src/dotty/tools/dottydoc/util/syntax.scala new file mode 100644 index 000000000..dd3d21f8d --- /dev/null +++ b/dottydoc/src/dotty/tools/dottydoc/util/syntax.scala @@ -0,0 +1,16 @@ +package dotty.tools +package dottydoc +package util + +import dotc.core.Contexts.Context +import dotc.core.Comments._ +import model.Package +import core.ContextDottydoc + +object syntax { + implicit class ContextWithContextDottydoc(val ctx: Context) extends AnyVal { + def docbase: ContextDottydoc = ctx.docCtx.getOrElse { + throw new IllegalStateException("DocBase must be set before running dottydoc phases") + }.asInstanceOf[ContextDottydoc] + } +} diff --git a/dottydoc/test/BaseTest.scala b/dottydoc/test/BaseTest.scala index 46a24c579..303c476a5 100644 --- a/dottydoc/test/BaseTest.scala +++ b/dottydoc/test/BaseTest.scala @@ -1,13 +1,14 @@ package dotty.tools package dottydoc -import dotc.core.Contexts -import Contexts.{ Context, ContextBase, FreshContext } +import dotc.core.Contexts.{ Context, ContextBase, FreshContext } +import dotc.core.Comments.{ ContextDoc, ContextDocstrings } import dotc.util.SourceFile import dotc.core.Phases.Phase import dotc.typer.FrontEnd -import dottydoc.core.DocASTPhase +import dottydoc.core.{ DocASTPhase, ContextDottydoc } import model.Package +import dotty.tools.dottydoc.util.syntax._ trait DottyTest { dotty.tools.dotc.parsing.Scanners // initialize keywords @@ -19,6 +20,7 @@ trait DottyTest { ctx.setSetting(ctx.settings.language, List("Scala2")) ctx.setSetting(ctx.settings.YkeepComments, true) ctx.setSetting(ctx.settings.YnoInline, true) + ctx.setProperty(ContextDoc, new ContextDottydoc) base.initialize()(ctx) ctx } @@ -28,7 +30,7 @@ trait DottyTest { List(new Phase { def phaseName = "assertionPhase" override def run(implicit ctx: Context): Unit = - assertion(ctx.docbase.packages[Package].toMap) + assertion(ctx.docbase.packages) }) :: Nil override def phases = diff --git a/dottydoc/test/ConstructorTest.scala b/dottydoc/test/ConstructorTest.scala index 8aa883022..44a05acad 100644 --- a/dottydoc/test/ConstructorTest.scala +++ b/dottydoc/test/ConstructorTest.scala @@ -22,7 +22,7 @@ class Constructors extends DottyTest { checkSources(source :: Nil) { packages => packages("scala") match { - case PackageImpl(_, List(cls: Class), _, _) => + case PackageImpl(_, _, List(cls: Class), _, _) => cls.constructors.headOption match { case Some(ParamListImpl(NamedReference("str", _, false, false) :: Nil, false) :: Nil) => // success! @@ -44,7 +44,7 @@ class Constructors extends DottyTest { checkSources(source :: Nil) { packages => packages("scala") match { - case PackageImpl(_, List(cls: Class), _, _) => + case PackageImpl(_, _, List(cls: Class), _, _) => cls.constructors match { case ( ParamListImpl(NamedReference("str1", _, false, false) :: Nil, false) :: @@ -69,7 +69,7 @@ class Constructors extends DottyTest { checkSources(source :: Nil) { packages => packages("scala") match { - case PackageImpl(_, List(cls: Class), _, _) => + case PackageImpl(_, _, List(cls: Class), _, _) => cls.constructors match { case ( ParamListImpl(NamedReference("str1", _, false, false) :: Nil, false) :: @@ -101,7 +101,7 @@ class Constructors extends DottyTest { checkSources(source :: Nil) { packages => packages("scala") match { - case PackageImpl(_, List(cls: Class), _, _) => + case PackageImpl(_, _, List(cls: Class), _, _) => cls.constructors match { case ( ParamListImpl(NamedReference("main", _, false, false) :: Nil, false) :: Nil @@ -139,7 +139,7 @@ class Constructors extends DottyTest { checkSources(source :: Nil) { packages => packages("scala") match { - case PackageImpl(_, List(cls: CaseClass, obj: Object), _, _) => + case PackageImpl(_, _, List(cls: CaseClass, obj: Object), _, _) => cls.constructors match { case ( ParamListImpl(NamedReference("main", _, false, false) :: Nil, false) :: Nil @@ -172,7 +172,7 @@ class Constructors extends DottyTest { checkSources(source :: Nil) { packages => packages("scala") match { - case PackageImpl(_, List(trt: Trait), _, _) => + case PackageImpl(_, _, List(trt: Trait), _, _) => trt.traitParams match { case ParamListImpl(NamedReference("main", _, false, false) :: Nil, false) :: Nil => case _ => @@ -199,7 +199,7 @@ class Constructors extends DottyTest { checkSources(source :: Nil) { packages => packages("scala") match { - case PackageImpl(_, List(cc: CaseClass, _, cls: Class, trt: Trait), _, _) => + case PackageImpl(_, _, List(cc: CaseClass, _, cls: Class, trt: Trait), _, _) => import model.json._ lazy val incorrectJson = s"The json generated for:\n$actualSource\n\nIs not correct" assert(cc.json.contains(s""""constructors":[[{"list":[{"title":"main""""), incorrectJson) diff --git a/dottydoc/test/PackageStructure.scala b/dottydoc/test/PackageStructure.scala index 00caaa2c0..4e7006bfe 100644 --- a/dottydoc/test/PackageStructure.scala +++ b/dottydoc/test/PackageStructure.scala @@ -29,7 +29,7 @@ class PackageStructure extends DottyTest { checkSources(source1 :: source2 :: Nil) { packages => packages("scala") match { - case PackageImpl(_, List(tA, tB), _, _) => + case PackageImpl(_, _, List(tA, tB), _, _) => assert( tA.name == "A" && tB.name == "B", s"trait A had name '${tA.name}' and trait B had name '${tB.name}'" @@ -62,8 +62,9 @@ class PackageStructure extends DottyTest { checkSources(source1 :: source2 :: Nil) { packages => packages("scala") match { case PackageImpl( + _, "scala", - List(PackageImpl("scala.collection", List(tA, tB), _, _)), + List(PackageImpl(_, "scala.collection", List(tA, tB), _, _)), _, _ ) => assert( @@ -76,7 +77,7 @@ class PackageStructure extends DottyTest { } packages("scala.collection") match { - case PackageImpl("scala.collection", List(tA, tB), _, _) => + case PackageImpl(_, "scala.collection", List(tA, tB), _, _) => assert( tA.name == "A" && tB.name == "B", s"trait A had name '${tA.name}' and trait B had name '${tB.name}'" diff --git a/dottydoc/test/UsecaseTest.scala b/dottydoc/test/UsecaseTest.scala new file mode 100644 index 000000000..b5d47e4b6 --- /dev/null +++ b/dottydoc/test/UsecaseTest.scala @@ -0,0 +1,234 @@ +package dotty.tools +package dottydoc + +import org.junit.Test +import org.junit.Assert._ + +import dotc.util.SourceFile +import model._ +import model.internal._ +import model.references._ +import util.syntax._ + +class UsecaseTest extends DottyTest { + @Test def simpleUsecase = { + val source = new SourceFile( + "DefWithUseCase.scala", + """ + |package scala + | + |trait Test[A] { + | /** Definition with a "disturbing" signature + | * + | * @usecase def foo: A + | */ + | def foo[B]: A => B + |} + """.stripMargin + ) + + checkSources(source :: Nil) { packages => + packages("scala") match { + case PackageImpl(_, _, List(trt: Trait), _, _) => + val List(foo: Def) = trt.members + + assert(foo.comment.isDefined, "Lost comment in transformations") + + val returnValue = foo.returnValue match { + case ref: TypeReference => ref.title + case _ => + assert( + false, + "Incorrect return value after usecase transformation" + ) + "" + } + + assert( + foo.typeParams.isEmpty, + "Type parameters were not stripped by usecase" + ) + assert(returnValue == "A", "Incorrect return type after usecase") + + assert(foo.name == "foo", s"Incorrect name after transform: ${foo.name}") + } + } + } + + @Test def simpleUsecaseAddedArg = { + val source = new SourceFile( + "DefWithUseCase.scala", + """ + |package scala + | + |trait Test[A] { + | /** Definition with a "disturbing" signature + | * + | * @usecase def foo(a: A): A + | */ + | def foo[B]: A => B + |} + """.stripMargin + ) + + checkSources(source :: Nil) { packages => + packages("scala") match { + case PackageImpl(_, _, List(trt: Trait), _, _) => + val List(foo: Def) = trt.members + + val returnValue = foo.returnValue match { + case ref: TypeReference => ref.title + case _ => + assert( + false, + "Incorrect return value after usecase transformation" + ) + "" + } + + assert( + foo.typeParams.isEmpty, + "Type parameters were not stripped by usecase" + ) + assert(returnValue == "A", "Incorrect return type after usecase") + assert( + foo.paramLists.head.list.head.title == "a", + "Incorrect parameter to function after usecase transformation" + ) + assert(foo.name == "foo", s"Incorrect name after transform: ${foo.name}") + } + } + } + + @Test def simpleTparamUsecase = { + val source = new SourceFile( + "DefWithUseCase.scala", + """ + |package scala + | + |trait Test[A] { + | /** Definition with a "disturbing" signature + | * + | * @usecase def foo[C]: A + | */ + | def foo[B]: A => B + |} + """.stripMargin + ) + + checkSources(source :: Nil) { packages => + packages("scala") match { + case PackageImpl(_, _, List(trt: Trait), _, _) => + val List(foo: Def) = trt.members + + val returnValue = foo.returnValue match { + case ref: TypeReference => ref.title + case _ => + assert( + false, + "Incorrect return value after usecase transformation" + ) + "" + } + + assert( + foo.typeParams.nonEmpty, + "Type parameters were incorrectly stripped by usecase" + ) + + assert(foo.typeParams.head == "C", "Incorrectly switched tparam") + assert(returnValue == "A", "Incorrect return type after usecase") + + assert(foo.name == "foo", s"Incorrect name after transform: ${foo.name}") + } + } + } + + @Test def expandColl = { + val source = new SourceFile( + "ExpandColl.scala", + """ + |package scala + | + |/** The trait $Coll + | * + | * @define Coll Iterable + | */ + |trait Iterable[A] { + | /** Definition with a "disturbing" signature + | * + | * @usecase def map[B](f: A => B): $Coll[B] + | */ + | def map[B, M[B]](f: A => B): M[B] = ??? + |} + """.stripMargin + ) + + checkSources(source :: Nil) { packages => + packages("scala") match { + case PackageImpl(_, _, List(trt: Trait), _, _) => + val List(map: Def) = trt.members + + val returnValue = map.returnValue match { + case ref: TypeReference => ref.title + case _ => + assert( + false, + "Incorrect return value after usecase transformation" + ) + "" + } + + assert( + returnValue == "Iterable", + "Incorrect return type after usecase transformation" + ) + } + } + } + + @Test def checkStripping = { + val source = new SourceFile( + "CheckStripping.scala", + """ + |package scala + | + |/** The trait $Coll + | * + | * @define Coll Iterable + | */ + |trait Iterable[A] { + | /** Definition with a "disturbing" signature + | * + | * @usecase def map[B](f: A => B): $Coll[B] + | */ + | def map[B, M[B]](f: A => B): M[B] = ??? + |} + """.stripMargin + ) + + checkSources(source :: Nil) { packages => + packages("scala") match { + case PackageImpl(_, _, List(trt: Trait), _, _) => + val List(map: Def) = trt.members + assert(map.comment.isDefined, "Lost comment in transformations") + + val docstr = ctx.docbase.docstring(map.symbol).get.raw + assert( + !docstr.contains("@usecase"), + s"Comment should not contain usecase after stripping, but was:\n$docstr" + ) + } + } + } + + @Test def checkIterator = + checkFiles("./scala-scala/src/library/scala/collection/Iterator.scala" :: Nil) { _ => + // success if typer throws no errors! :) + } + + @Test def checkIterableLike = + checkFiles("./scala-scala/src/library/scala/collection/IterableLike.scala" :: Nil) { _ => + // success if typer throws no errors! :) + } +} diff --git a/src/dotty/tools/dotc/ast/Trees.scala b/src/dotty/tools/dotc/ast/Trees.scala index 6986e40e7..70701ecd7 100644 --- a/src/dotty/tools/dotc/ast/Trees.scala +++ b/src/dotty/tools/dotc/ast/Trees.scala @@ -4,7 +4,7 @@ package ast import core._ import Types._, Names._, Flags._, util.Positions._, Contexts._, Constants._, SymDenotations._, Symbols._ -import Denotations._, StdNames._ +import Denotations._, StdNames._, Comments._ import annotation.tailrec import language.higherKinds import collection.IndexedSeqOptimized @@ -15,7 +15,6 @@ import printing.Printer import util.{Stats, Attachment, Property, DotClass} import annotation.unchecked.uncheckedVariance import language.implicitConversions -import parsing.Scanners.Comment object Trees { diff --git a/src/dotty/tools/dotc/core/Comments.scala b/src/dotty/tools/dotc/core/Comments.scala new file mode 100644 index 000000000..1cf5aec38 --- /dev/null +++ b/src/dotty/tools/dotc/core/Comments.scala @@ -0,0 +1,458 @@ +package dotty.tools +package dotc +package core + +import ast.{ untpd, tpd } +import Decorators._, Symbols._, Contexts._, Flags.EmptyFlags +import util.SourceFile +import util.Positions._ +import util.CommentParsing._ +import util.Property.Key +import parsing.Parsers.Parser + +object Comments { + val ContextDoc = new Key[ContextDocstrings] + + /** Decorator for getting docbase out of context */ + implicit class CommentsContext(val ctx: Context) extends AnyVal { + def docCtx: Option[ContextDocstrings] = ctx.property(ContextDoc) + } + + /** Context for Docstrings, contains basic functionality for getting + * docstrings via `Symbol` and expanding templates + */ + class ContextDocstrings { + import scala.collection.mutable + + private[this] val _docstrings: mutable.Map[Symbol, Comment] = + mutable.Map.empty + + val templateExpander = new CommentExpander + + def docstrings: Map[Symbol, Comment] = _docstrings.toMap + + def docstring(sym: Symbol): Option[Comment] = _docstrings.get(sym) + + def addDocstring(sym: Symbol, doc: Option[Comment]): Unit = + doc.map(d => _docstrings += (sym -> d)) + } + + /** A `Comment` contains the unformatted docstring as well as a position + * + * The `Comment` contains functionality to create versions of itself without + * `@usecase` sections as well as functionality to map the `raw` docstring + */ + abstract case class Comment(pos: Position, raw: String) { self => + def isExpanded: Boolean + + def usecases: List[UseCase] + + val isDocComment = raw.startsWith("/**") + + def expand(f: String => String): Comment = new Comment(pos, f(raw)) { + val isExpanded = true + val usecases = self.usecases + } + + def withUsecases(implicit ctx: Context): Comment = new Comment(pos, stripUsecases) { + val isExpanded = self.isExpanded + val usecases = parseUsecases + } + + private[this] lazy val stripUsecases: String = + removeSections(raw, "@usecase", "@define") + + private[this] def parseUsecases(implicit ctx: Context): List[UseCase] = + if (!raw.startsWith("/**")) + List.empty[UseCase] + else + tagIndex(raw) + .filter { startsWithTag(raw, _, "@usecase") } + .map { case (start, end) => decomposeUseCase(start, end) } + + /** Turns a usecase section into a UseCase, with code changed to: + * {{{ + * // From: + * def foo: A + * // To: + * def foo: A = ??? + * }}} + */ + private[this] def decomposeUseCase(start: Int, end: Int)(implicit ctx: Context): UseCase = { + def subPos(start: Int, end: Int) = + if (pos == NoPosition) NoPosition + else { + val start1 = pos.start + start + val end1 = pos.end + end + pos withStart start1 withPoint start1 withEnd end1 + } + + val codeStart = skipWhitespace(raw, start + "@usecase".length) + val codeEnd = skipToEol(raw, codeStart) + val code = raw.substring(codeStart, codeEnd) + " = ???" + val codePos = subPos(codeStart, codeEnd) + val commentStart = skipLineLead(raw, codeEnd + 1) min end + val commentStr = "/** " + raw.substring(commentStart, end) + "*/" + val commentPos = subPos(commentStart, end) + + UseCase(Comment(commentPos, commentStr), code, codePos) + } + } + + object Comment { + def apply(pos: Position, raw: String, expanded: Boolean = false, usc: List[UseCase] = Nil)(implicit ctx: Context): Comment = + new Comment(pos, raw) { + val isExpanded = expanded + val usecases = usc + } + } + + abstract case class UseCase(comment: Comment, code: String, codePos: Position) { + /** Set by typer */ + var tpdCode: tpd.DefDef = _ + + def untpdCode: untpd.Tree + } + + object UseCase { + def apply(comment: Comment, code: String, codePos: Position)(implicit ctx: Context) = + new UseCase(comment, code, codePos) { + val untpdCode = { + val tree = new Parser(new SourceFile("<usecase>", code)).localDef(codePos.start, EmptyFlags) + + tree match { + case tree: untpd.DefDef => + val newName = (tree.name.show + "$" + codePos + "$doc").toTermName + untpd.DefDef(newName, tree.tparams, tree.vparamss, tree.tpt, tree.rhs) + case _ => + ctx.error("proper definition was not found in `@usecase`", codePos) + tree + } + } + } + } + + /** + * Port of DocComment.scala from nsc + * @author Martin Odersky + * @author Felix Mulder + */ + class CommentExpander { + import dotc.config.Printers.dottydoc + import scala.collection.mutable + + def expand(sym: Symbol, site: Symbol)(implicit ctx: Context): String = { + val parent = if (site != NoSymbol) site else sym + defineVariables(parent) + expandedDocComment(sym, parent) + } + + /** The cooked doc comment of symbol `sym` after variable expansion, or "" if missing. + * + * @param sym The symbol for which doc comment is returned + * @param site The class for which doc comments are generated + * @throws ExpansionLimitExceeded when more than 10 successive expansions + * of the same string are done, which is + * interpreted as a recursive variable definition. + */ + def expandedDocComment(sym: Symbol, site: Symbol, docStr: String = "")(implicit ctx: Context): String = { + // when parsing a top level class or module, use the (module-)class itself to look up variable definitions + val parent = if ((sym.is(Flags.Module) || sym.isClass) && site.is(Flags.Package)) sym + else site + expandVariables(cookedDocComment(sym, docStr), sym, parent) + } + + private def template(raw: String): String = + removeSections(raw, "@define") + + private def defines(raw: String): List[String] = { + val sections = tagIndex(raw) + val defines = sections filter { startsWithTag(raw, _, "@define") } + val usecases = sections filter { startsWithTag(raw, _, "@usecase") } + val end = startTag(raw, (defines ::: usecases).sortBy(_._1)) + + defines map { case (start, end) => raw.substring(start, end) } + } + + private def replaceInheritDocToInheritdoc(docStr: String): String = + docStr.replaceAll("""\{@inheritDoc\p{Zs}*\}""", "@inheritdoc") + + /** The cooked doc comment of an overridden symbol */ + protected def superComment(sym: Symbol)(implicit ctx: Context): Option[String] = + allInheritedOverriddenSymbols(sym).iterator map (x => cookedDocComment(x)) find (_ != "") + + private val cookedDocComments = mutable.HashMap[Symbol, String]() + + /** The raw doc comment of symbol `sym`, minus usecase and define sections, augmented by + * missing sections of an inherited doc comment. + * If a symbol does not have a doc comment but some overridden version of it does, + * the doc comment of the overridden version is copied instead. + */ + def cookedDocComment(sym: Symbol, docStr: String = "")(implicit ctx: Context): String = cookedDocComments.getOrElseUpdate(sym, { + var ownComment = + if (docStr.length == 0) ctx.docCtx.flatMap(_.docstring(sym).map(c => template(c.raw))).getOrElse("") + else template(docStr) + ownComment = replaceInheritDocToInheritdoc(ownComment) + + superComment(sym) match { + case None => + // SI-8210 - The warning would be false negative when this symbol is a setter + if (ownComment.indexOf("@inheritdoc") != -1 && ! sym.isSetter) + dottydoc.println(s"${sym.pos}: the comment for ${sym} contains @inheritdoc, but no parent comment is available to inherit from.") + ownComment.replaceAllLiterally("@inheritdoc", "<invalid inheritdoc annotation>") + case Some(sc) => + if (ownComment == "") sc + else expandInheritdoc(sc, merge(sc, ownComment, sym), sym) + } + }) + + private def isMovable(str: String, sec: (Int, Int)): Boolean = + startsWithTag(str, sec, "@param") || + startsWithTag(str, sec, "@tparam") || + startsWithTag(str, sec, "@return") + + def merge(src: String, dst: String, sym: Symbol, copyFirstPara: Boolean = false): String = { + val srcSections = tagIndex(src) + val dstSections = tagIndex(dst) + val srcParams = paramDocs(src, "@param", srcSections) + val dstParams = paramDocs(dst, "@param", dstSections) + val srcTParams = paramDocs(src, "@tparam", srcSections) + val dstTParams = paramDocs(dst, "@tparam", dstSections) + val out = new StringBuilder + var copied = 0 + var tocopy = startTag(dst, dstSections dropWhile (!isMovable(dst, _))) + + if (copyFirstPara) { + val eop = // end of comment body (first para), which is delimited by blank line, or tag, or end of comment + (findNext(src, 0)(src.charAt(_) == '\n')) min startTag(src, srcSections) + out append src.substring(0, eop).trim + copied = 3 + tocopy = 3 + } + + def mergeSection(srcSec: Option[(Int, Int)], dstSec: Option[(Int, Int)]) = dstSec match { + case Some((start, end)) => + if (end > tocopy) tocopy = end + case None => + srcSec match { + case Some((start1, end1)) => { + out append dst.substring(copied, tocopy).trim + out append "\n" + copied = tocopy + out append src.substring(start1, end1).trim + } + case None => + } + } + + //TODO: enable this once you know how to get `sym.paramss` + /* + for (params <- sym.paramss; param <- params) + mergeSection(srcParams get param.name.toString, dstParams get param.name.toString) + for (tparam <- sym.typeParams) + mergeSection(srcTParams get tparam.name.toString, dstTParams get tparam.name.toString) + + mergeSection(returnDoc(src, srcSections), returnDoc(dst, dstSections)) + mergeSection(groupDoc(src, srcSections), groupDoc(dst, dstSections)) + */ + + if (out.length == 0) dst + else { + out append dst.substring(copied) + out.toString + } + } + + /** + * Expand inheritdoc tags + * - for the main comment we transform the inheritdoc into the super variable, + * and the variable expansion can expand it further + * - for the param, tparam and throws sections we must replace comments on the spot + * + * This is done separately, for two reasons: + * 1. It takes longer to run compared to merge + * 2. The inheritdoc annotation should not be used very often, as building the comment from pieces severely + * impacts performance + * + * @param parent The source (or parent) comment + * @param child The child (overriding member or usecase) comment + * @param sym The child symbol + * @return The child comment with the inheritdoc sections expanded + */ + def expandInheritdoc(parent: String, child: String, sym: Symbol): String = + if (child.indexOf("@inheritdoc") == -1) + child + else { + val parentSections = tagIndex(parent) + val childSections = tagIndex(child) + val parentTagMap = sectionTagMap(parent, parentSections) + val parentNamedParams = Map() + + ("@param" -> paramDocs(parent, "@param", parentSections)) + + ("@tparam" -> paramDocs(parent, "@tparam", parentSections)) + + ("@throws" -> paramDocs(parent, "@throws", parentSections)) + + val out = new StringBuilder + + def replaceInheritdoc(childSection: String, parentSection: => String) = + if (childSection.indexOf("@inheritdoc") == -1) + childSection + else + childSection.replaceAllLiterally("@inheritdoc", parentSection) + + def getParentSection(section: (Int, Int)): String = { + + def getSectionHeader = extractSectionTag(child, section) match { + case param@("@param"|"@tparam"|"@throws") => param + " " + extractSectionParam(child, section) + case other => other + } + + def sectionString(param: String, paramMap: Map[String, (Int, Int)]): String = + paramMap.get(param) match { + case Some(section) => + // Cleanup the section tag and parameter + val sectionTextBounds = extractSectionText(parent, section) + cleanupSectionText(parent.substring(sectionTextBounds._1, sectionTextBounds._2)) + case None => + dottydoc.println(s"""${sym.pos}: the """" + getSectionHeader + "\" annotation of the " + sym + + " comment contains @inheritdoc, but the corresponding section in the parent is not defined.") + "<invalid inheritdoc annotation>" + } + + child.substring(section._1, section._1 + 7) match { + case param@("@param "|"@tparam"|"@throws") => + sectionString(extractSectionParam(child, section), parentNamedParams(param.trim)) + case _ => + sectionString(extractSectionTag(child, section), parentTagMap) + } + } + + def mainComment(str: String, sections: List[(Int, Int)]): String = + if (str.trim.length > 3) + str.trim.substring(3, startTag(str, sections)) + else + "" + + // Append main comment + out.append("/**") + out.append(replaceInheritdoc(mainComment(child, childSections), mainComment(parent, parentSections))) + + // Append sections + for (section <- childSections) + out.append(replaceInheritdoc(child.substring(section._1, section._2), getParentSection(section))) + + out.append("*/") + out.toString + } + + protected def expandVariables(initialStr: String, sym: Symbol, site: Symbol)(implicit ctx: Context): String = { + val expandLimit = 10 + + def expandInternal(str: String, depth: Int): String = { + if (depth >= expandLimit) + throw new ExpansionLimitExceeded(str) + + val out = new StringBuilder + var copied, idx = 0 + // excluding variables written as \$foo so we can use them when + // necessary to document things like Symbol#decode + def isEscaped = idx > 0 && str.charAt(idx - 1) == '\\' + while (idx < str.length) { + if ((str charAt idx) != '$' || isEscaped) + idx += 1 + else { + val vstart = idx + idx = skipVariable(str, idx + 1) + def replaceWith(repl: String) = { + out append str.substring(copied, vstart) + out append repl + copied = idx + } + variableName(str.substring(vstart + 1, idx)) match { + case "super" => + superComment(sym) foreach { sc => + val superSections = tagIndex(sc) + replaceWith(sc.substring(3, startTag(sc, superSections))) + for (sec @ (start, end) <- superSections) + if (!isMovable(sc, sec)) out append sc.substring(start, end) + } + case "" => idx += 1 + case vname => + lookupVariable(vname, site) match { + case Some(replacement) => replaceWith(replacement) + case None => + dottydoc.println(s"Variable $vname undefined in comment for $sym in $site") + } + } + } + } + if (out.length == 0) str + else { + out append str.substring(copied) + expandInternal(out.toString, depth + 1) + } + } + + // We suppressed expanding \$ throughout the recursion, and now we + // need to replace \$ with $ so it looks as intended. + expandInternal(initialStr, 0).replaceAllLiterally("""\$""", "$") + } + + def defineVariables(sym: Symbol)(implicit ctx: Context) = { + val Trim = "(?s)^[\\s&&[^\n\r]]*(.*?)\\s*$".r + + val raw = ctx.docCtx.flatMap(_.docstring(sym).map(_.raw)).getOrElse("") + defs(sym) ++= defines(raw).map { + str => { + val start = skipWhitespace(str, "@define".length) + val (key, value) = str.splitAt(skipVariable(str, start)) + key.drop(start) -> value + } + } map { + case (key, Trim(value)) => + variableName(key) -> value.replaceAll("\\s+\\*+$", "") + } + } + + /** Maps symbols to the variable -> replacement maps that are defined + * in their doc comments + */ + private val defs = mutable.HashMap[Symbol, Map[String, String]]() withDefaultValue Map() + + /** Lookup definition of variable. + * + * @param vble The variable for which a definition is searched + * @param site The class for which doc comments are generated + */ + def lookupVariable(vble: String, site: Symbol)(implicit ctx: Context): Option[String] = site match { + case NoSymbol => None + case _ => + val searchList = + if (site.flags.is(Flags.Module)) site :: site.info.baseClasses + else site.info.baseClasses + + searchList collectFirst { case x if defs(x) contains vble => defs(x)(vble) } match { + case Some(str) if str startsWith "$" => lookupVariable(str.tail, site) + case res => res orElse lookupVariable(vble, site.owner) + } + } + + /** The position of the raw doc comment of symbol `sym`, or NoPosition if missing + * If a symbol does not have a doc comment but some overridden version of it does, + * the position of the doc comment of the overridden version is returned instead. + */ + def docCommentPos(sym: Symbol)(implicit ctx: Context): Position = + ctx.docCtx.flatMap(_.docstring(sym).map(_.pos)).getOrElse(NoPosition) + + /** A version which doesn't consider self types, as a temporary measure: + * an infinite loop has broken out between superComment and cookedDocComment + * since r23926. + */ + private def allInheritedOverriddenSymbols(sym: Symbol)(implicit ctx: Context): List[Symbol] = { + if (!sym.owner.isClass) Nil + else sym.allOverriddenSymbols.toList.filter(_ != NoSymbol) //TODO: could also be `sym.owner.allOverrid..` + //else sym.owner.ancestors map (sym overriddenSymbol _) filter (_ != NoSymbol) + } + + class ExpansionLimitExceeded(str: String) extends Exception + } +} diff --git a/src/dotty/tools/dotc/core/Contexts.scala b/src/dotty/tools/dotc/core/Contexts.scala index 313ea3124..5c9fdaf88 100644 --- a/src/dotty/tools/dotc/core/Contexts.scala +++ b/src/dotty/tools/dotc/core/Contexts.scala @@ -13,6 +13,7 @@ import Scopes._ import NameOps._ import Uniques._ import SymDenotations._ +import Comments._ import Flags.ParamAccessor import util.Positions._ import ast.Trees._ @@ -29,7 +30,6 @@ import printing._ import config.{Settings, ScalaSettings, Platform, JavaPlatform, SJSPlatform} import language.implicitConversions import DenotTransformers.DenotTransformer -import parsing.Scanners.Comment import util.Property.Key import xsbti.AnalysisCallback @@ -537,9 +537,6 @@ object Contexts { /** The symbol loaders */ val loaders = new SymbolLoaders - /** Documentation base */ - val docbase = new DocBase - /** The platform, initialized by `initPlatform()`. */ private var _platform: Platform = _ @@ -578,32 +575,6 @@ object Contexts { } } - class DocBase { - private[this] val _docstrings: mutable.Map[Symbol, Comment] = - mutable.Map.empty - - def docstring(sym: Symbol): Option[Comment] = _docstrings.get(sym) - - def addDocstring(sym: Symbol, doc: Option[Comment]): Unit = - doc.map(d => _docstrings += (sym -> d)) - - /* - * Dottydoc places instances of `Package` in this map - but we do not want - * to depend on `dottydoc` for the compiler, as such this is defined as a - * map of `String -> AnyRef` - */ - private[this] val _packages: mutable.Map[String, AnyRef] = mutable.Map.empty - def packages[A]: mutable.Map[String, A] = _packages.asInstanceOf[mutable.Map[String, A]] - - /** Should perhaps factorize this into caches that get flushed */ - private var _defs: Map[Symbol, Set[Symbol]] = Map.empty - def defs(sym: Symbol): Set[Symbol] = _defs.get(sym).getOrElse(Set.empty) - - def addDef(s: Symbol, d: Symbol): Unit = _defs = (_defs + { - s -> _defs.get(s).map(xs => xs + d).getOrElse(Set(d)) - }) - } - /** The essential mutable state of a context base, collected into a common class */ class ContextState { // Symbols state diff --git a/src/dotty/tools/dotc/core/SymDenotations.scala b/src/dotty/tools/dotc/core/SymDenotations.scala index 969d09c3e..5a5eacd18 100644 --- a/src/dotty/tools/dotc/core/SymDenotations.scala +++ b/src/dotty/tools/dotc/core/SymDenotations.scala @@ -3,7 +3,7 @@ package dotc package core import Periods._, Contexts._, Symbols._, Denotations._, Names._, NameOps._, Annotations._ -import Types._, Flags._, Decorators._, DenotTransformers._, StdNames._, Scopes._ +import Types._, Flags._, Decorators._, DenotTransformers._, StdNames._, Scopes._, Comments._ import NameOps._ import Scopes.Scope import collection.mutable @@ -1541,13 +1541,15 @@ object SymDenotations { /** Enter a symbol in given `scope` without potentially replacing the old copy. */ def enterNoReplace(sym: Symbol, scope: MutableScope)(implicit ctx: Context): Unit = { - + def isUsecase = ctx.docCtx.isDefined && sym.name.show.takeRight(4) == "$doc" require( (sym.denot.flagsUNSAFE is Private) || !(this is Frozen) || (scope ne this.unforcedDecls) || sym.hasAnnotation(defn.ScalaStaticAnnot) || - sym.name.isInlineAccessor) + sym.name.isInlineAccessor || + isUsecase) + scope.enter(sym) if (myMemberFingerPrint != FingerPrint.unknown) diff --git a/src/dotty/tools/dotc/parsing/Parsers.scala b/src/dotty/tools/dotc/parsing/Parsers.scala index 0a25bf801..9aadf0c61 100644 --- a/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/src/dotty/tools/dotc/parsing/Parsers.scala @@ -19,10 +19,10 @@ import StdNames._ import util.Positions._ import Constants._ import ScriptParsers._ +import Comments._ import scala.annotation.{tailrec, switch} import util.DotClass import rewrite.Rewrites.patch -import Scanners.Comment object Parsers { diff --git a/src/dotty/tools/dotc/parsing/Scanners.scala b/src/dotty/tools/dotc/parsing/Scanners.scala index e16aa670f..60003d098 100644 --- a/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/src/dotty/tools/dotc/parsing/Scanners.scala @@ -3,7 +3,7 @@ package dotc package parsing import core.Names._, core.Contexts._, core.Decorators._, util.Positions._ -import core.StdNames._ +import core.StdNames._, core.Comments._ import util.SourceFile import java.lang.Character.isDigit import scala.reflect.internal.Chars._ @@ -22,10 +22,6 @@ object Scanners { /** An undefined offset */ val NoOffset: Offset = -1 - case class Comment(pos: Position, chrs: String) { - def isDocComment = chrs.startsWith("/**") - } - type Token = Int trait TokenData { diff --git a/src/dotty/tools/dotc/typer/Docstrings.scala b/src/dotty/tools/dotc/typer/Docstrings.scala new file mode 100644 index 000000000..370844e65 --- /dev/null +++ b/src/dotty/tools/dotc/typer/Docstrings.scala @@ -0,0 +1,56 @@ +package dotty.tools +package dotc +package typer + +import core._ +import Contexts._, Symbols._, Decorators._, Comments._ +import util.Positions._ +import ast.tpd + +trait Docstrings { self: Typer => + + /** The Docstrings typer will handle the expansion of `@define` and + * `@inheritdoc` if there is a `DocContext` present as a property in the + * supplied `ctx`. + * + * It will also type any `@usecase` available in function definitions. + */ + def cookComments(syms: List[Symbol], owner: Symbol)(implicit ctx: Context): Unit = + ctx.docCtx.foreach { docbase => + val relevantSyms = syms.filter(docbase.docstring(_).isDefined) + relevantSyms.foreach { sym => + expandParentDocs(sym) + val usecases = docbase.docstring(sym).map(_.usecases).getOrElse(Nil) + + usecases.foreach { usecase => + enterSymbol(createSymbol(usecase.untpdCode)) + + typedStats(usecase.untpdCode :: Nil, owner) match { + case List(df: tpd.DefDef) => usecase.tpdCode = df + case _ => ctx.error("`@usecase` was not a valid definition", usecase.codePos) + } + } + } + } + + private def expandParentDocs(sym: Symbol)(implicit ctx: Context): Unit = + ctx.docCtx.foreach { docCtx => + docCtx.docstring(sym).foreach { cmt => + def expandDoc(owner: Symbol): Unit = if (!cmt.isExpanded) { + val tplExp = docCtx.templateExpander + tplExp.defineVariables(sym) + + val newCmt = cmt + .expand(tplExp.expandedDocComment(sym, owner, _)) + .withUsecases + + docCtx.addDocstring(sym, Some(newCmt)) + } + + if (sym ne NoSymbol) { + expandParentDocs(sym.owner) + expandDoc(sym.owner) + } + } + } +} diff --git a/src/dotty/tools/dotc/typer/Namer.scala b/src/dotty/tools/dotc/typer/Namer.scala index 2e714ab6d..4f4278468 100644 --- a/src/dotty/tools/dotc/typer/Namer.scala +++ b/src/dotty/tools/dotc/typer/Namer.scala @@ -4,7 +4,7 @@ package typer import core._ import ast._ -import Trees._, Constants._, StdNames._, Scopes._, Denotations._ +import Trees._, Constants._, StdNames._, Scopes._, Denotations._, Comments._ import Contexts._, Symbols._, Types._, SymDenotations._, Names._, NameOps._, Flags._, Decorators._ import ast.desugar, ast.desugar._ import ProtoTypes._ @@ -456,7 +456,8 @@ class Namer { typer: Typer => def setDocstring(sym: Symbol, tree: Tree)(implicit ctx: Context) = tree match { - case t: MemberDef => ctx.docbase.addDocstring(sym, t.rawComment) + case t: MemberDef if t.rawComment.isDefined => + ctx.docCtx.foreach(_.addDocstring(sym, t.rawComment)) case _ => () } diff --git a/src/dotty/tools/dotc/typer/Typer.scala b/src/dotty/tools/dotc/typer/Typer.scala index 3aff69bdb..3e3bb32f5 100644 --- a/src/dotty/tools/dotc/typer/Typer.scala +++ b/src/dotty/tools/dotc/typer/Typer.scala @@ -11,6 +11,7 @@ import Scopes._ import Denotations._ import ProtoTypes._ import Contexts._ +import Comments._ import Symbols._ import Types._ import SymDenotations._ @@ -57,7 +58,7 @@ object Typer { assert(tree.pos.exists, s"position not set for $tree # ${tree.uniqueId}") } -class Typer extends Namer with TypeAssigner with Applications with Implicits with Dynamic with Checking { +class Typer extends Namer with TypeAssigner with Applications with Implicits with Dynamic with Checking with Docstrings { import Typer._ import tpd.{cpy => _, _} @@ -1181,6 +1182,7 @@ class Typer extends Namer with TypeAssigner with Applications with Implicits wit // function types so no dependencies on parameters are allowed. tpt1 = tpt1.withType(avoid(tpt1.tpe, vparamss1.flatMap(_.map(_.symbol)))) } + assignType(cpy.DefDef(ddef)(name, tparams1, vparamss1, tpt1, rhs1), sym) //todo: make sure dependent method types do not depend on implicits or by-name params } @@ -1244,6 +1246,10 @@ class Typer extends Namer with TypeAssigner with Applications with Implicits wit val self1 = typed(self)(ctx.outer).asInstanceOf[ValDef] // outer context where class members are not visible val dummy = localDummy(cls, impl) val body1 = typedStats(impl.body, dummy)(inClassContext(self1.symbol)) + + // Expand comments and type usecases + cookComments(body1.map(_.symbol), self1.symbol)(localContext(cdef, cls).setNewScope) + checkNoDoubleDefs(cls) val impl1 = cpy.Template(impl)(constr1, parents1, self1, body1) .withType(dummy.nonMemberTermRef) diff --git a/dottydoc/src/dotty/tools/dottydoc/model/comment/CommentUtils.scala b/src/dotty/tools/dotc/util/CommentParsing.scala index e5307bd3c..cc790d683 100644 --- a/dottydoc/src/dotty/tools/dottydoc/model/comment/CommentUtils.scala +++ b/src/dotty/tools/dotc/util/CommentParsing.scala @@ -3,15 +3,17 @@ * @author Martin Odersky * @author Felix Mulder */ - -package dotty.tools -package dottydoc -package model -package comment - -import scala.reflect.internal.Chars._ - -object CommentUtils { +package dotty.tools.dotc.util + +/** The comment parsing in `dotc` is used by both the comment cooking and the + * dottydoc tool. + * + * The comment cooking is used to expand comments with `@inheritdoc` and + * `@define` annotations. The rest of the comment is untouched and later + * handled by dottydoc. + */ +object CommentParsing { + import scala.reflect.internal.Chars._ /** Returns index of string `str` following `start` skipping longest * sequence of whitespace characters characters (but no newlines) @@ -221,4 +223,17 @@ object CommentUtils { result } + + def removeSections(raw: String, xs: String*): String = { + val sections = tagIndex(raw) + + val toBeRemoved = for { + section <- xs + lines = sections filter { startsWithTag(raw, _, section) } + } yield lines + + val end = startTag(raw, toBeRemoved.flatten.sortBy(_._1).toList) + + if (end == raw.length - 2) raw else raw.substring(0, end) + "*/" + } } diff --git a/test/test/DottyDocParsingTests.scala b/test/test/DottyDocParsingTests.scala index ed89c6114..8522cdae3 100644 --- a/test/test/DottyDocParsingTests.scala +++ b/test/test/DottyDocParsingTests.scala @@ -14,7 +14,7 @@ class DottyDocParsingTests extends DottyDocTest { checkFrontend(source) { case PackageDef(_, Seq(c: TypeDef)) => - assert(c.rawComment.map(_.chrs) == None, "Should not have a comment, mainly used for exhaustive tests") + assert(c.rawComment.map(_.raw) == None, "Should not have a comment, mainly used for exhaustive tests") } } @@ -29,7 +29,7 @@ class DottyDocParsingTests extends DottyDocTest { checkFrontend(source) { case PackageDef(_, Seq(t @ TypeDef(name, _))) if name.toString == "Class" => - checkDocString(t.rawComment.map(_.chrs), "/** Hello world! */") + checkDocString(t.rawComment.map(_.raw), "/** Hello world! */") } } @@ -44,7 +44,7 @@ class DottyDocParsingTests extends DottyDocTest { checkFrontend(source) { case PackageDef(_, Seq(t @ TypeDef(name, _))) if name.toString == "Class" => - checkDocString(t.rawComment.map(_.chrs), "/** Hello /* multiple open */ world! */") + checkDocString(t.rawComment.map(_.raw), "/** Hello /* multiple open */ world! */") } } @Test def multipleClassesInPackage = { @@ -62,8 +62,8 @@ class DottyDocParsingTests extends DottyDocTest { checkCompile("frontend", source) { (_, ctx) => ctx.compilationUnit.untpdTree match { case PackageDef(_, Seq(c1 @ TypeDef(_,_), c2 @ TypeDef(_,_))) => { - checkDocString(c1.rawComment.map(_.chrs), "/** Class1 docstring */") - checkDocString(c2.rawComment.map(_.chrs), "/** Class2 docstring */") + checkDocString(c1.rawComment.map(_.raw), "/** Class1 docstring */") + checkDocString(c2.rawComment.map(_.raw), "/** Class2 docstring */") } } } @@ -77,7 +77,7 @@ class DottyDocParsingTests extends DottyDocTest { """.stripMargin checkFrontend(source) { - case PackageDef(_, Seq(t @ TypeDef(_,_))) => checkDocString(t.rawComment.map(_.chrs), "/** Class without package */") + case PackageDef(_, Seq(t @ TypeDef(_,_))) => checkDocString(t.rawComment.map(_.raw), "/** Class without package */") } } @@ -85,7 +85,7 @@ class DottyDocParsingTests extends DottyDocTest { val source = "/** Trait docstring */\ntrait Trait" checkFrontend(source) { - case PackageDef(_, Seq(t @ TypeDef(_,_))) => checkDocString(t.rawComment.map(_.chrs), "/** Trait docstring */") + case PackageDef(_, Seq(t @ TypeDef(_,_))) => checkDocString(t.rawComment.map(_.raw), "/** Trait docstring */") } } @@ -101,8 +101,8 @@ class DottyDocParsingTests extends DottyDocTest { checkFrontend(source) { case PackageDef(_, Seq(t1 @ TypeDef(_,_), t2 @ TypeDef(_,_))) => { - checkDocString(t1.rawComment.map(_.chrs), "/** Trait1 docstring */") - checkDocString(t2.rawComment.map(_.chrs), "/** Trait2 docstring */") + checkDocString(t1.rawComment.map(_.raw), "/** Trait1 docstring */") + checkDocString(t2.rawComment.map(_.raw), "/** Trait2 docstring */") } } } @@ -127,10 +127,10 @@ class DottyDocParsingTests extends DottyDocTest { checkFrontend(source) { case PackageDef(_, Seq(t1 @ TypeDef(_,_), c2 @ TypeDef(_,_), cc3 @ TypeDef(_,_), _, ac4 @ TypeDef(_,_))) => { - checkDocString(t1.rawComment.map(_.chrs), "/** Trait1 docstring */") - checkDocString(c2.rawComment.map(_.chrs), "/** Class2 docstring */") - checkDocString(cc3.rawComment.map(_.chrs), "/** CaseClass3 docstring */") - checkDocString(ac4.rawComment.map(_.chrs), "/** AbstractClass4 docstring */") + checkDocString(t1.rawComment.map(_.raw), "/** Trait1 docstring */") + checkDocString(c2.rawComment.map(_.raw), "/** Class2 docstring */") + checkDocString(cc3.rawComment.map(_.raw), "/** CaseClass3 docstring */") + checkDocString(ac4.rawComment.map(_.raw), "/** AbstractClass4 docstring */") } } } @@ -147,9 +147,9 @@ class DottyDocParsingTests extends DottyDocTest { checkFrontend(source) { case PackageDef(_, Seq(outer @ TypeDef(_, tpl @ Template(_,_,_,_)))) => { - checkDocString(outer.rawComment.map(_.chrs), "/** Outer docstring */") + checkDocString(outer.rawComment.map(_.raw), "/** Outer docstring */") tpl.body match { - case (inner @ TypeDef(_,_)) :: _ => checkDocString(inner.rawComment.map(_.chrs), "/** Inner docstring */") + case (inner @ TypeDef(_,_)) :: _ => checkDocString(inner.rawComment.map(_.raw), "/** Inner docstring */") case _ => assert(false, "Couldn't find inner class") } } @@ -171,10 +171,10 @@ class DottyDocParsingTests extends DottyDocTest { checkFrontend(source) { case PackageDef(_, Seq(o1 @ TypeDef(_, tpl @ Template(_,_,_,_)), o2 @ TypeDef(_,_))) => { - checkDocString(o1.rawComment.map(_.chrs), "/** Outer1 docstring */") - checkDocString(o2.rawComment.map(_.chrs), "/** Outer2 docstring */") + checkDocString(o1.rawComment.map(_.raw), "/** Outer1 docstring */") + checkDocString(o2.rawComment.map(_.raw), "/** Outer2 docstring */") tpl.body match { - case (inner @ TypeDef(_,_)) :: _ => checkDocString(inner.rawComment.map(_.chrs), "/** Inner docstring */") + case (inner @ TypeDef(_,_)) :: _ => checkDocString(inner.rawComment.map(_.raw), "/** Inner docstring */") case _ => assert(false, "Couldn't find inner class") } } @@ -196,9 +196,9 @@ class DottyDocParsingTests extends DottyDocTest { checkFrontend(source) { case p @ PackageDef(_, Seq(o1: MemberDef[Untyped], o2: MemberDef[Untyped])) => { assertEquals(o1.name.toString, "Object1") - checkDocString(o1.rawComment.map(_.chrs), "/** Object1 docstring */") + checkDocString(o1.rawComment.map(_.raw), "/** Object1 docstring */") assertEquals(o2.name.toString, "Object2") - checkDocString(o2.rawComment.map(_.chrs), "/** Object2 docstring */") + checkDocString(o2.rawComment.map(_.raw), "/** Object2 docstring */") } } } @@ -223,12 +223,12 @@ class DottyDocParsingTests extends DottyDocTest { checkFrontend(source) { case p @ PackageDef(_, Seq(o1: ModuleDef, o2: ModuleDef)) => { assert(o1.name.toString == "Object1") - checkDocString(o1.rawComment.map(_.chrs), "/** Object1 docstring */") + checkDocString(o1.rawComment.map(_.raw), "/** Object1 docstring */") assert(o2.name.toString == "Object2") - checkDocString(o2.rawComment.map(_.chrs), "/** Object2 docstring */") + checkDocString(o2.rawComment.map(_.raw), "/** Object2 docstring */") o2.impl.body match { - case _ :: (inner @ TypeDef(_,_)) :: _ => checkDocString(inner.rawComment.map(_.chrs), "/** Inner docstring */") + case _ :: (inner @ TypeDef(_,_)) :: _ => checkDocString(inner.rawComment.map(_.raw), "/** Inner docstring */") case _ => assert(false, "Couldn't find inner class") } } @@ -257,14 +257,14 @@ class DottyDocParsingTests extends DottyDocTest { import dotty.tools.dotc.ast.untpd._ checkFrontend(source) { case PackageDef(_, Seq(p: ModuleDef)) => { - checkDocString(p.rawComment.map(_.chrs), "/** Package object docstring */") + checkDocString(p.rawComment.map(_.raw), "/** Package object docstring */") p.impl.body match { case (b: TypeDef) :: (t: TypeDef) :: (o: ModuleDef) :: Nil => { - checkDocString(b.rawComment.map(_.chrs), "/** Boo docstring */") - checkDocString(t.rawComment.map(_.chrs), "/** Trait docstring */") - checkDocString(o.rawComment.map(_.chrs), "/** InnerObject docstring */") - checkDocString(o.impl.body.head.asInstanceOf[TypeDef].rawComment.map(_.chrs), "/** InnerClass docstring */") + checkDocString(b.rawComment.map(_.raw), "/** Boo docstring */") + checkDocString(t.rawComment.map(_.raw), "/** Trait docstring */") + checkDocString(o.rawComment.map(_.raw), "/** InnerObject docstring */") + checkDocString(o.impl.body.head.asInstanceOf[TypeDef].rawComment.map(_.raw), "/** InnerClass docstring */") } case _ => assert(false, "Incorrect structure inside package object") } @@ -284,7 +284,7 @@ class DottyDocParsingTests extends DottyDocTest { import dotty.tools.dotc.ast.untpd._ checkFrontend(source) { case PackageDef(_, Seq(c: TypeDef)) => - checkDocString(c.rawComment.map(_.chrs), "/** Real comment */") + checkDocString(c.rawComment.map(_.raw), "/** Real comment */") } } @@ -303,7 +303,7 @@ class DottyDocParsingTests extends DottyDocTest { import dotty.tools.dotc.ast.untpd._ checkFrontend(source) { case PackageDef(_, Seq(c: TypeDef)) => - checkDocString(c.rawComment.map(_.chrs), "/** Real comment */") + checkDocString(c.rawComment.map(_.raw), "/** Real comment */") } } @@ -329,9 +329,9 @@ class DottyDocParsingTests extends DottyDocTest { case PackageDef(_, Seq(o: ModuleDef)) => { o.impl.body match { case (v1: MemberDef) :: (v2: MemberDef) :: (v3: MemberDef) :: Nil => { - checkDocString(v1.rawComment.map(_.chrs), "/** val1 */") - checkDocString(v2.rawComment.map(_.chrs), "/** val2 */") - checkDocString(v3.rawComment.map(_.chrs), "/** val3 */") + checkDocString(v1.rawComment.map(_.raw), "/** val1 */") + checkDocString(v2.rawComment.map(_.raw), "/** val2 */") + checkDocString(v3.rawComment.map(_.raw), "/** val3 */") } case _ => assert(false, "Incorrect structure inside object") } @@ -361,9 +361,9 @@ class DottyDocParsingTests extends DottyDocTest { case PackageDef(_, Seq(o: ModuleDef)) => { o.impl.body match { case (v1: MemberDef) :: (v2: MemberDef) :: (v3: MemberDef) :: Nil => { - checkDocString(v1.rawComment.map(_.chrs), "/** var1 */") - checkDocString(v2.rawComment.map(_.chrs), "/** var2 */") - checkDocString(v3.rawComment.map(_.chrs), "/** var3 */") + checkDocString(v1.rawComment.map(_.raw), "/** var1 */") + checkDocString(v2.rawComment.map(_.raw), "/** var2 */") + checkDocString(v3.rawComment.map(_.raw), "/** var3 */") } case _ => assert(false, "Incorrect structure inside object") } @@ -393,9 +393,9 @@ class DottyDocParsingTests extends DottyDocTest { case PackageDef(_, Seq(o: ModuleDef)) => { o.impl.body match { case (v1: MemberDef) :: (v2: MemberDef) :: (v3: MemberDef) :: Nil => { - checkDocString(v1.rawComment.map(_.chrs), "/** def1 */") - checkDocString(v2.rawComment.map(_.chrs), "/** def2 */") - checkDocString(v3.rawComment.map(_.chrs), "/** def3 */") + checkDocString(v1.rawComment.map(_.raw), "/** def1 */") + checkDocString(v2.rawComment.map(_.raw), "/** def2 */") + checkDocString(v3.rawComment.map(_.raw), "/** def3 */") } case _ => assert(false, "Incorrect structure inside object") } @@ -425,9 +425,9 @@ class DottyDocParsingTests extends DottyDocTest { case PackageDef(_, Seq(o: ModuleDef)) => { o.impl.body match { case (v1: MemberDef) :: (v2: MemberDef) :: (v3: MemberDef) :: Nil => { - checkDocString(v1.rawComment.map(_.chrs), "/** type1 */") - checkDocString(v2.rawComment.map(_.chrs), "/** type2 */") - checkDocString(v3.rawComment.map(_.chrs), "/** type3 */") + checkDocString(v1.rawComment.map(_.raw), "/** type1 */") + checkDocString(v2.rawComment.map(_.raw), "/** type2 */") + checkDocString(v3.rawComment.map(_.raw), "/** type3 */") } case _ => assert(false, "Incorrect structure inside object") } @@ -451,7 +451,7 @@ class DottyDocParsingTests extends DottyDocTest { case PackageDef(_, Seq(o: ModuleDef)) => o.impl.body match { case (foo: MemberDef) :: Nil => - expectNoDocString(foo.rawComment.map(_.chrs)) + expectNoDocString(foo.rawComment.map(_.raw)) case _ => assert(false, "Incorrect structure inside object") } } @@ -468,7 +468,7 @@ class DottyDocParsingTests extends DottyDocTest { import dotty.tools.dotc.ast.untpd._ checkFrontend(source) { case p @ PackageDef(_, Seq(_, c: TypeDef)) => - checkDocString(c.rawComment.map(_.chrs), "/** Class1 */") + checkDocString(c.rawComment.map(_.raw), "/** Class1 */") } } @@ -483,7 +483,7 @@ class DottyDocParsingTests extends DottyDocTest { import dotty.tools.dotc.ast.untpd._ checkFrontend(source) { case p @ PackageDef(_, Seq(c: TypeDef)) => - checkDocString(c.rawComment.map(_.chrs), "/** Class1 */") + checkDocString(c.rawComment.map(_.raw), "/** Class1 */") } } } /* End class */ |