Scaladoc diff-firendly output

Scaladoc can create raw content files that we can easily diff and spot
any modifications. There is a cool project by Stefan Zeiger to export
the scaladoc model in JSON, but with the language and scaladoc being so
quick to evolve, it'll be a pain to properly maintain. In the long-run,
the plan is to sample a couple of raw files on each build and email me
the diff. If I spot anything that may be wrong I can fix it, revert the
commit or at least file a bug.

For now, .html.raw files are generated on-demand, using
  ant -Dscaladoc.raw.output="yes" <targets>

Also added a script that will do the job of diff-ing.

Review by @jsuereth.

Conflicts:

	src/compiler/scala/tools/nsc/doc/Settings.scala

7 files changed, 102 insertions, 25 deletions

diff --git a/build.xml b/build.xml
index 33ebbfe377..e6b23f268f 100644
--- a/build.xml
+++ b/build.xml
@@ -2064,6 +2064,9 @@ DOCUMENTATION
     <!-- Compute the URL and show it -->
     <property name="scaladoc.url" value="https://github.com/scala/scala/tree/${scaladoc.git.commit}/src"/>
     <echo message="Scaladoc will point to ${scaladoc.url} for source files."/>
+
+    <!-- Unless set with -Dscaladoc.raw.output, it won't be activated -->
+    <property name="scaladoc.raw.output" value="no"/>
   </target>
 
   <target name="docs.pre-lib" depends="docs.start">
@@ -2091,7 +2094,7 @@ DOCUMENTATION
       classpathref="pack.classpath"
       addparams="${scalac.args.all}"
       docRootContent="${src.dir}/library/rootdoc.txt"
-      implicits="on" diagrams="on">
+      implicits="on" diagrams="on" rawOutput="${scaladoc.raw.output}">
       <src>
         <files includes="${src.dir}/actors-migration"/>
         <files includes="${src.dir}/actors"/>
@@ -2175,7 +2178,7 @@ DOCUMENTATION
       srcdir="${src.dir}/compiler"
       docRootContent="${src.dir}/compiler/rootdoc.txt"
       addparams="${scalac.args.all}"
-      implicits="on" diagrams="on">
+      implicits="on" diagrams="on" rawOutput="${scaladoc.raw.output}">
       <include name="**/*.scala"/>
     </scaladoc>
     <touch file="${build-docs.dir}/compiler.complete" verbose="no"/>
@@ -2197,7 +2200,7 @@ DOCUMENTATION
       classpathref="pack.classpath"
       srcdir="${src.dir}/jline/src/main/java"
       addparams="${scalac.args.all}"
-      implicits="on" diagrams="on">
+      implicits="on" diagrams="on" rawOutput="${scaladoc.raw.output}">
       <include name="**/*.scala"/>
       <include name="**/*.java"/>
     </scaladoc>
@@ -2221,7 +2224,7 @@ DOCUMENTATION
       classpathref="pack.classpath"
       srcdir="${src.dir}/scalap"
       addparams="${scalac.args.all}"
-      implicits="on" diagrams="on">
+      implicits="on" diagrams="on" rawOutput="${scaladoc.raw.output}">
       <include name="**/*.scala"/>
     </scaladoc>
     <touch file="${build-docs.dir}/scalap.complete" verbose="no"/>
@@ -2243,7 +2246,7 @@ DOCUMENTATION
       classpathref="pack.classpath"
       srcdir="${src.dir}/partest"
       addparams="${scalac.args.all}"
-      implicits="on" diagrams="on">
+      implicits="on" diagrams="on" rawOutput="${scaladoc.raw.output}">
       <include name="**/*.scala"/>
     </scaladoc>
     <touch file="${build-docs.dir}/partest.complete" verbose="no"/>
@@ -2265,7 +2268,7 @@ DOCUMENTATION
       classpathref="pack.classpath"
       srcdir="${src.dir}/continuations/plugin"
       addparams="${scalac.args.all}"
-      implicits="on" diagrams="on">
+      implicits="on" diagrams="on" rawOutput="${scaladoc.raw.output}">
       <include name="**/*.scala"/>
     </scaladoc>
     <touch file="${build-docs.dir}/continuations-plugin.complete" verbose="no"/>
@@ -2287,7 +2290,7 @@ DOCUMENTATION
       classpathref="pack.classpath"
       srcdir="${src.dir}/actors-migration"
       addparams="${scalac.args.all}"
-      implicits="on" diagrams="on">
+      implicits="on" diagrams="on" rawOutput="${scaladoc.raw.output}">
       <include name="**/*.scala"/>
     </scaladoc>
     <touch file="${build-docs.dir}/actors-migration.complete" verbose="no"/>
diff --git a/src/compiler/scala/tools/ant/Scaladoc.scala b/src/compiler/scala/tools/ant/Scaladoc.scala
index daa08ef8a7..2cada92c1e 100644
--- a/src/compiler/scala/tools/ant/Scaladoc.scala
+++ b/src/compiler/scala/tools/ant/Scaladoc.scala
@@ -150,6 +150,9 @@ class Scaladoc extends ScalaMatchingTask {
   /** Instruct the scaladoc tool to use the binary given to create diagrams */
   private var docDiagramsDotPath: Option[String] = None
 
+  /** Instruct the scaladoc to produce textual ouput from html pages, for easy diff-ing */
+  private var docRawOutput: Boolean = false
+
 
 /*============================================================================*\
 **                             Properties setters                             **
@@ -419,6 +422,11 @@ class Scaladoc extends ScalaMatchingTask {
   def setDiagramsDotPath(input: String) =
     docDiagramsDotPath = Some(input)
 
+  /** Set the `rawOutput` bit so Scaladoc also outputs text from each html file
+   *  @param input One of the flags `yes/no` or `on/off`. Default if no/off. */
+  def setRawOutput(input: String) =
+    docRawOutput = Flag.getBooleanValue(input, "rawOutput")
+
 /*============================================================================*\
 **                             Properties getters                             **
 \*============================================================================*/
@@ -616,6 +624,7 @@ class Scaladoc extends ScalaMatchingTask {
     docSettings.docImplicitsShowAll.value = docImplicitsShowAll
     docSettings.docDiagrams.value = docDiagrams
     docSettings.docDiagramsDebug.value = docDiagramsDebug
+    docSettings.docRawOutput.value = docRawOutput
     if(!docDiagramsDotPath.isEmpty) docSettings.docDiagramsDotPath.value = docDiagramsDotPath.get
 
     if (!docgenerator.isEmpty) docSettings.docgenerator.value = docgenerator.get
diff --git a/src/compiler/scala/tools/nsc/doc/Settings.scala b/src/compiler/scala/tools/nsc/doc/Settings.scala
index 4458889d55..d7b61bc129 100644
--- a/src/compiler/scala/tools/nsc/doc/Settings.scala
+++ b/src/compiler/scala/tools/nsc/doc/Settings.scala
@@ -120,6 +120,11 @@ class Settings(error: String => Unit) extends scala.tools.nsc.Settings(error) {
     "dot" // by default, just pick up the system-wide dot
   )
 
+  val docRawOutput = BooleanSetting (
+    "-raw-output",
+    "For each html file, create another .html.raw file containing only the text. (can be used for quickly diffing two scaladoc outputs)"
+  )
+
   // Somewhere slightly before r18708 scaladoc stopped building unless the
   // self-type check was suppressed.  I hijacked the slotted-for-removal-anyway
   // suppress-vt-warnings option and renamed it for this purpose.
diff --git a/src/compiler/scala/tools/nsc/doc/html/HtmlPage.scala b/src/compiler/scala/tools/nsc/doc/html/HtmlPage.scala
index e3da8bddea..4c9215f923 100644
--- a/src/compiler/scala/tools/nsc/doc/html/HtmlPage.scala
+++ b/src/compiler/scala/tools/nsc/doc/html/HtmlPage.scala
@@ -13,7 +13,7 @@ import comment._
 import xml.{XML, NodeSeq}
 import xml.dtd.{DocType, PublicID}
 import scala.collection._
-import java.nio.channels.Channels
+import java.io.Writer
 
 /** An html page that is part of a Scaladoc site.
   * @author David Bernard
@@ -52,17 +52,19 @@ abstract class HtmlPage extends Page { thisPage =>
         </head>
         { body }
       </html>
-    val fos = createFileOutputStream(site)
-    val w = Channels.newWriter(fos.getChannel, site.encoding)
-    try {
+
+    writeFile(site) { (w: Writer) =>
       w.write("<?xml version='1.0' encoding='" + site.encoding + "'?>\n")
       w.write(doctype.toString + "\n")
       w.write(xml.Xhtml.toXhtml(html))
     }
-    finally {
-      w.close()
-      fos.close()
-    }
+
+    if (site.universe.settings.docRawOutput.value)
+      writeFile(site, ".raw") {
+        // we're only interested in the body, as this will go into the diff
+        _.write(body.text)
+      }
+
     //XML.save(pageFile.getPath, html, site.encoding, xmlDecl = false, doctype = doctype)
   }
 
diff --git a/src/compiler/scala/tools/nsc/doc/html/Page.scala b/src/compiler/scala/tools/nsc/doc/html/Page.scala
index c5bf3e0e37..72b62dd482 100644
--- a/src/compiler/scala/tools/nsc/doc/html/Page.scala
+++ b/src/compiler/scala/tools/nsc/doc/html/Page.scala
@@ -8,6 +8,8 @@ package scala.tools.nsc.doc.html
 import scala.tools.nsc.doc.model._
 import java.io.{FileOutputStream, File}
 import scala.reflect.NameTransformer
+import java.nio.channels.Channels
+import java.io.Writer
 
 abstract class Page {
   thisPage =>
@@ -20,8 +22,8 @@ abstract class Page {
 
   def absoluteLinkTo(path: List[String]) = path.reverse.mkString("/")
 
-  def createFileOutputStream(site: HtmlFactory) = {
-    val file = new File(site.siteRoot, absoluteLinkTo(thisPage.path))
+  def createFileOutputStream(site: HtmlFactory, suffix: String = "") = {
+    val file = new File(site.siteRoot, absoluteLinkTo(thisPage.path) + suffix)
     val folder = file.getParentFile
     if (! folder.exists) {
       folder.mkdirs
@@ -29,6 +31,18 @@ abstract class Page {
     new FileOutputStream(file.getPath)
   }
 
+  def writeFile(site: HtmlFactory, suffix: String = "")(fn: Writer => Unit) = {
+    val fos = createFileOutputStream(site, suffix)
+    val w = Channels.newWriter(fos.getChannel, site.encoding)
+    try {
+      fn(w)
+    }
+    finally {
+      w.close()
+      fos.close()
+    }
+  }
+
   /** Writes this page as a file. The file's location is relative to the
     * generator's site root, and the encoding is also defined by the generator.
     * @param generator The generator that is writing this page. */
diff --git a/src/compiler/scala/tools/nsc/doc/html/page/IndexScript.scala b/src/compiler/scala/tools/nsc/doc/html/page/IndexScript.scala
index 7edd4937c4..c5fe4da17a 100644
--- a/src/compiler/scala/tools/nsc/doc/html/page/IndexScript.scala
+++ b/src/compiler/scala/tools/nsc/doc/html/page/IndexScript.scala
@@ -15,14 +15,8 @@ class IndexScript(universe: doc.Universe, index: doc.Index) extends Page {
   def path = List("index.js")
 
   override def writeFor(site: HtmlFactory) {
-    val stream = createFileOutputStream(site)
-    val writer = Channels.newWriter(stream.getChannel, site.encoding)
-    try {
-      writer.write("Index.PACKAGES = " + packages.toString() + ";")
-    }
-    finally {
-      writer.close
-      stream.close
+    writeFile(site) {
+      _.write("Index.PACKAGES = " + packages.toString() + ";")
     }
   }
 
diff --git a/tools/scaladoc-compare b/tools/scaladoc-compare
new file mode 100755
index 0000000000..74fbfd1dd4
--- /dev/null
+++ b/tools/scaladoc-compare
@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Script to compare scaladoc raw files. For an explanation read the next echos.
+#
+
+if [ $# -ne 2 ]
+then
+  echo
+  echo "scaladoc-compare will compare the scaladoc-generated pages in two different locations and output the diff"
+  echo "it's main purpose is to track changes to scaladoc and prevent updates that break things."
+  echo
+  echo "This script is meant to be used with the scaladoc -raw-output option, as it compares .html.raw files "
+  echo "instead of markup-heavy .html files."
+  echo
+  echo "Script usage $0 <new api files path> <old api files path>"
+  echo "   eg: $0 build/scaladoc/library build/scaladoc-prev/library | less"
+  echo 
+  exit 1
+fi
+
+NEW_PATH=$1
+OLD_PATH=$2
+
+FILES=`find $NEW_PATH -name '*.html.raw'`
+if [ "$FILES" == "" ]
+then
+  echo "No .html.raw files found in $NEW_PATH!"
+  exit 1
+fi
+
+for NEW_FILE in $FILES
+do
+  OLD_FILE=${NEW_FILE/$NEW_PATH/$OLD_PATH}
+  if [ -f $OLD_FILE ]
+  then
+    #echo $NEW_FILE" => "$OLD_FILE
+    DIFF=`diff -q -w $NEW_FILE $OLD_FILE 2>&1`
+    if [ "$DIFF" != "" ]
+    then
+      # Redo the full diff
+      echo "$NEW_FILE:"
+      diff -w $NEW_FILE $OLD_FILE 2>&1
+      echo -e "\n\n"
+    fi
+  else
+    echo -e "$NEW_FILE: No corresponding file (expecting $OLD_FILE)\n\n"
+  fi
+done
+
+echo Done.