Implement shortening of Markdown AST for summaries

5 files changed, 163 insertions, 35 deletions

diff --git a/doc-tool/src/dotty/tools/dottydoc/core/ContextDottydoc.scala b/doc-tool/src/dotty/tools/dottydoc/core/ContextDottydoc.scala
index f2122572b..963c72bc9 100644
--- a/doc-tool/src/dotty/tools/dottydoc/core/ContextDottydoc.scala
+++ b/doc-tool/src/dotty/tools/dottydoc/core/ContextDottydoc.scala
@@ -25,7 +25,6 @@ class ContextDottydoc extends ContextDocstrings {
     s -> _defs.get(s).map(xs => xs + d).getOrElse(Set(d))
   })
 
-
   val markdownOptions: DataHolder =
     new MutableDataSet()
       .setFrom(ParserEmulationFamily.KRAMDOWN.getOptions)
diff --git a/doc-tool/src/dotty/tools/dottydoc/model/comment/Comment.scala b/doc-tool/src/dotty/tools/dottydoc/model/comment/Comment.scala
index 8d927cb28..842746338 100644
--- a/doc-tool/src/dotty/tools/dottydoc/model/comment/Comment.scala
+++ b/doc-tool/src/dotty/tools/dottydoc/model/comment/Comment.scala
@@ -68,28 +68,27 @@ object MarkdownComment extends util.MemberLookup {
       }
     }
 
-    val parsedBody = parsed.body.fromMarkdown(ent)
     Comment(
-      body                    = parsedBody,
-      short                   = parsedBody,
-      authors                 = filterEmpty(parsed.authors).map(_.fromMarkdown(ent)),
-      see                     = filterEmpty(parsed.see).map(_.fromMarkdown(ent)),
-      result                  = single("@result", parsed.result).map(_.fromMarkdown(ent)),
-      throws                  = linkedExceptions(parsed.throws).mapValues(_.fromMarkdown(ent)),
-      valueParams             = filterEmpty(parsed.valueParams).mapValues(_.fromMarkdown(ent)),
-      typeParams              = filterEmpty(parsed.typeParams).mapValues(_.fromMarkdown(ent)),
-      version                 = single("@version", parsed.version).map(_.fromMarkdown(ent)),
-      since                   = single("@since", parsed.since).map(_.fromMarkdown(ent)),
-      todo                    = filterEmpty(parsed.todo).map(_.fromMarkdown(ent)),
-      deprecated              = single("@deprecated", parsed.deprecated, filter = false).map(_.fromMarkdown(ent)),
-      note                    = filterEmpty(parsed.note).map(_.fromMarkdown(ent)),
-      example                 = filterEmpty(parsed.example).map(_.fromMarkdown(ent)),
-      constructor             = single("@constructor", parsed.constructor).map(_.fromMarkdown(ent)),
-      group                   = single("@group", parsed.group).map(_.fromMarkdown(ent)),
-      groupDesc               = filterEmpty(parsed.groupDesc).mapValues(_.fromMarkdown(ent)),
-      groupNames              = filterEmpty(parsed.groupNames).mapValues(_.fromMarkdown(ent)),
-      groupPrio               = filterEmpty(parsed.groupPrio).mapValues(_.fromMarkdown(ent)),
-      hideImplicitConversions = filterEmpty(parsed.hideImplicitConversions).map(_.fromMarkdown(ent))
+      body                    = parsed.body.toMarkdownString(ent),
+      short                   = parsed.body.toMarkdown(ent).shortenAndShow,
+      authors                 = filterEmpty(parsed.authors).map(_.toMarkdownString(ent)),
+      see                     = filterEmpty(parsed.see).map(_.toMarkdownString(ent)),
+      result                  = single("@result", parsed.result).map(_.toMarkdownString(ent)),
+      throws                  = linkedExceptions(parsed.throws).mapValues(_.toMarkdownString(ent)),
+      valueParams             = filterEmpty(parsed.valueParams).mapValues(_.toMarkdownString(ent)),
+      typeParams              = filterEmpty(parsed.typeParams).mapValues(_.toMarkdownString(ent)),
+      version                 = single("@version", parsed.version).map(_.toMarkdownString(ent)),
+      since                   = single("@since", parsed.since).map(_.toMarkdownString(ent)),
+      todo                    = filterEmpty(parsed.todo).map(_.toMarkdownString(ent)),
+      deprecated              = single("@deprecated", parsed.deprecated, filter = false).map(_.toMarkdownString(ent)),
+      note                    = filterEmpty(parsed.note).map(_.toMarkdownString(ent)),
+      example                 = filterEmpty(parsed.example).map(_.toMarkdownString(ent)),
+      constructor             = single("@constructor", parsed.constructor).map(_.toMarkdownString(ent)),
+      group                   = single("@group", parsed.group).map(_.toMarkdownString(ent)),
+      groupDesc               = filterEmpty(parsed.groupDesc).mapValues(_.toMarkdownString(ent)),
+      groupNames              = filterEmpty(parsed.groupNames).mapValues(_.toMarkdownString(ent)),
+      groupPrio               = filterEmpty(parsed.groupPrio).mapValues(_.toMarkdownString(ent)),
+      hideImplicitConversions = filterEmpty(parsed.hideImplicitConversions).map(_.toMarkdownString(ent))
     )
   }
 
@@ -114,7 +113,7 @@ object WikiComment extends util.MemberLookup {
   def apply(parsed: ParsedComment, ent: Entity, pos: Position)(implicit ctx: Context): Comment = {
     val inlineToHtml = InlineToHtml(ent)
     val packages = ctx.docbase.packages
-    val parsedBody = parsed.body.toWiki(ent, packages, pos).wikiToString(ent)
+    val parsedBody = parsed.body.toWiki(ent, packages, pos).show(ent)
 
     def linkedExceptions(m: Map[String, Body]): Map[String, Body] = {
       m.map { case (targetStr, body) =>
@@ -131,7 +130,7 @@ object WikiComment extends util.MemberLookup {
     }
 
     def toWiki(str: String): Body = str.toWiki(ent, packages, pos)
-    def toString(body: Body): String = body.wikiToString(ent)
+    def toString(body: Body): String = body.show(ent)
 
     def filterEmpty(xs: List[String]): List[Body] =
       xs.map(toWiki).filterNot(_.blocks.isEmpty)
diff --git a/doc-tool/src/dotty/tools/dottydoc/model/comment/HtmlParsers.scala b/doc-tool/src/dotty/tools/dottydoc/model/comment/HtmlParsers.scala
index 24572925c..d3b695aac 100644
--- a/doc-tool/src/dotty/tools/dottydoc/model/comment/HtmlParsers.scala
+++ b/doc-tool/src/dotty/tools/dottydoc/model/comment/HtmlParsers.scala
@@ -8,22 +8,22 @@ import dotc.util.Positions._
 import dotty.tools.dottydoc.util.syntax._
 import util.MemberLookup
 
+import com.vladsch.flexmark.ast.{ Node => MarkdownNode }
+import com.vladsch.flexmark.html.HtmlRenderer
+import com.vladsch.flexmark.parser.Parser
+import com.vladsch.flexmark.util.sequence.CharSubSequence
+
 object HtmlParsers {
 
-  implicit class MarkdownToHtml(val text: String) extends AnyVal {
-    def fromMarkdown(origin: Entity)(implicit ctx: Context): String = {
-      import com.vladsch.flexmark.util.sequence.CharSubSequence
+  implicit class StringToMarkdown(val text: String) extends AnyVal {
+    def toMarkdown(origin: Entity)(implicit ctx: Context): MarkdownNode = {
       import com.vladsch.flexmark.ast.{ Link, Visitor, VisitHandler, NodeVisitor }
-      import com.vladsch.flexmark.parser.Parser
-      import com.vladsch.flexmark.html.HtmlRenderer
 
       val inlineToHtml = InlineToHtml(origin)
 
-      // TODO: split out into different step so that we can get a short summary
       val node = Parser.builder(ctx.docbase.markdownOptions)
         .build.parse(text)
 
-      implicit def toCharSeq(str: String) = CharSubSequence.of(str)
 
       def isOuter(url: String) =
         url.startsWith("http://") ||
@@ -49,14 +49,25 @@ object HtmlParsers {
           override def visit(link: Link) = {
             val linkUrl = link.getUrl.toString
             if (!isOuter(linkUrl) && !isRelative(linkUrl))
-              link.setUrl(queryToUrl(linkUrl))
+              link.setUrl(CharSubSequence.of(queryToUrl(linkUrl)))
           }
         })
       )
 
       linkVisitor.visit(node)
-      HtmlRenderer.builder(ctx.docbase.markdownOptions).build().render(node)
+      node
     }
+
+    def toMarkdownString(origin: Entity)(implicit ctx: Context): String =
+      toMarkdown(origin).show
+  }
+
+  implicit class MarkdownToHtml(val markdown: MarkdownNode) extends AnyVal {
+    def show(implicit ctx: Context): String =
+      HtmlRenderer.builder(ctx.docbase.markdownOptions).build().render(markdown)
+
+    def shortenAndShow(implicit ctx: Context): String =
+      (new MarkdownShortener).shorten(markdown).show
   }
 
   implicit class StringToWiki(val text: String) extends AnyVal {
@@ -65,7 +76,7 @@ object HtmlParsers {
   }
 
   implicit class BodyToHtml(val body: Body) extends AnyVal {
-    def wikiToString(origin: Entity): String = {
+    def show(origin: Entity): String = {
       val inlineToHtml = InlineToHtml(origin)
 
       def bodyToHtml(body: Body): String =
diff --git a/doc-tool/src/dotty/tools/dottydoc/model/comment/MarkdownShortener.scala b/doc-tool/src/dotty/tools/dottydoc/model/comment/MarkdownShortener.scala
new file mode 100644
index 000000000..841232e52
--- /dev/null
+++ b/doc-tool/src/dotty/tools/dottydoc/model/comment/MarkdownShortener.scala
@@ -0,0 +1,57 @@
+package dotty.tools
+package dottydoc
+package model
+package comment
+
+/** The `MarkdownShortener` takes a node and *mutates* it and all children so
+  * that the displayed length of the generated HTML doesn't exceeed `maxLen`.
+  * This number defaults to 150 characters.
+  *
+  * @note calling `shorten` **will** mutate the Markdown AST node.
+  */
+class MarkdownShortener {
+  import com.vladsch.flexmark.ast._
+
+  def shorten(node: Node, maxLen: Int = 150): Node = {
+    var len = 0
+
+    def count(node: Node, length: => Int, shortenOrUnlink: Int => Unit) = {
+      val remaining = math.max(maxLen - len, 0)
+      if (remaining == 0) node.unlink()
+      else {
+        if (length <= remaining) len += length
+        else {
+          shortenOrUnlink(remaining)
+          len = maxLen
+        }
+      }
+    }
+
+    val nodeVisitor = new NodeVisitor(
+      new VisitHandler(classOf[Text], new Visitor[Text] {
+        override def visit(node: Text) = count(
+          node,
+          node.getChars.length,
+          remaining => node.setChars(
+            node.getChars.subSequence(0, remaining).trimEnd.append("...")
+          )
+        )
+      }),
+      new VisitHandler(classOf[Code], new Visitor[Code] {
+        override def visit(node: Code) = count(
+          node,
+          node.getText.length,
+          remaining => node.setText(
+            node.getText.subSequence(0, remaining).trimEnd.append("...")
+          )
+        )
+      }),
+      new VisitHandler(classOf[Image], new Visitor[Image] {
+        override def visit(node: Image) = count(node, maxLen, _ => node.unlink())
+      })
+    )
+
+    nodeVisitor.visit(node)
+    node
+  }
+}
diff --git a/doc-tool/test/MarkdownTests.scala b/doc-tool/test/MarkdownTests.scala
index e8596b634..144acbdfb 100644
--- a/doc-tool/test/MarkdownTests.scala
+++ b/doc-tool/test/MarkdownTests.scala
@@ -237,11 +237,73 @@ class MarkdownTests extends DottyDocTest {
         .flatMap(_.comment.map(_.body))
         .get
         .trim
-        println(traitCmt)
 
         assertEquals(
           """|<pre><code class="language-scala">val x = 1 + 5
              |</code></pre>""".stripMargin, traitCmt)
     }
   }
+
+  @Test def docstringSummary = {
+    val source =
+      """
+      |package scala
+      |
+      |/** This
+      |  * ====
+      |  * is a short text [that](http://google.com) should not be more than a
+      |  * `few` lines long. This text *should* be shortened somewhere that is
+      |  * appropriate for the **ui**. Might be here, or there or somewhere
+      |  * else.
+      |  */
+      |trait HelloWorld
+      """.stripMargin
+
+    checkSource(source) { packages =>
+      val traitCmt =
+        packages("scala")
+        .children.find(_.path.mkString(".") == "scala.HelloWorld")
+        .flatMap(_.comment.map(_.short))
+        .get
+        .trim
+
+      assert(
+        traitCmt.endsWith("Might be here...\n</p>"),
+        s"""|docstring summary should strip the following docstring so that it ends in "Might be here..."
+            |
+            |$traitCmt""".stripMargin
+      )
+    }
+  }
+
+  @Test def docstringSummaryWithImage = {
+    val source =
+      """
+      |package scala
+      |
+      |/** This
+      |  * ====
+      |  * should quit before ![alt text](https://whatever.com/1.png "Img Text"),
+      |  * I shouldn't be visible.
+      |  */
+      |trait HelloWorld
+      """.stripMargin
+
+    checkSource(source) { packages =>
+      val traitCmt =
+        packages("scala")
+        .children.find(_.path.mkString(".") == "scala.HelloWorld")
+        .flatMap(_.comment.map(_.short))
+        .get
+        .trim
+
+      assert(
+        !traitCmt.contains("<img") &&
+        !traitCmt.contains("I shouldn't be visible."),
+        s"""|docstring summary shouldn't contain image, error in `MarkdownShortener.scala`
+            |
+            |$traitCmt""".stripMargin)
+    }
+
+  }
 }