Better explanation for how to use MIMA excludes.

This patch does a few things:
1. We have a file MimaExcludes.scala exclusively for excludes.
2. The test runner tells users about that file if a test fails.
3. I've added back the excludes used from 0.9->1.0. We should keep
   these in the project as an official audit trail of times where
   we decided to make exceptions.

Author: Patrick Wendell <pwendell@gmail.com>

Closes #937 from pwendell/mima and squashes the following commits:

7ee0db2 [Patrick Wendell] Better explanation for how to use MIMA excludes.

2 files changed, 85 insertions, 24 deletions

diff --git a/project/MimaBuild.scala b/project/MimaBuild.scala
index 182ca7615d..1477809943 100644
--- a/project/MimaBuild.scala
+++ b/project/MimaBuild.scala
@@ -15,21 +15,41 @@
  * limitations under the License.
  */
 
+import com.typesafe.tools.mima.core.{MissingTypesProblem, MissingClassProblem, ProblemFilters}
+import com.typesafe.tools.mima.core.ProblemFilters._
 import com.typesafe.tools.mima.plugin.MimaKeys.{binaryIssueFilters, previousArtifact}
 import com.typesafe.tools.mima.plugin.MimaPlugin.mimaDefaultSettings
 import sbt._
 
 object MimaBuild {
+  // Exclude a single class and its corresponding object
+  def excludeClass(className: String) = {
+    Seq(
+      excludePackage(className),
+      ProblemFilters.exclude[MissingClassProblem](className),
+      ProblemFilters.exclude[MissingTypesProblem](className),
+      excludePackage(className + "$"),
+      ProblemFilters.exclude[MissingClassProblem](className + "$"),
+      ProblemFilters.exclude[MissingTypesProblem](className + "$")
+    )
+  }
+  // Exclude a Spark class, that is in the package org.apache.spark
+  def excludeSparkClass(className: String) = {
+    excludeClass("org.apache.spark." + className)
+  }
+
+  // Exclude a Spark package, that is in the package org.apache.spark
+  def excludeSparkPackage(packageName: String) = {
+    excludePackage("org.apache.spark." + packageName)
+  }
 
   def ignoredABIProblems(base: File) = {
-    import com.typesafe.tools.mima.core._
-    import com.typesafe.tools.mima.core.ProblemFilters._
 
     // Excludes placed here will be used for all Spark versions
     val defaultExcludes = Seq()
 
     // Read package-private excludes from file
-    val excludeFilePath = (base.getAbsolutePath + "/.mima-excludes")
+    val excludeFilePath = (base.getAbsolutePath + "/.generated-mima-excludes")
     val excludeFile = file(excludeFilePath)
     val ignoredClasses: Seq[String] =
       if (!excludeFile.exists()) {
@@ -38,31 +58,11 @@ object MimaBuild {
         IO.read(excludeFile).split("\n")
       }
 
-    // Exclude a single class and its corresponding object
-    def excludeClass(className: String) = {
-      Seq(
-        excludePackage(className),
-        ProblemFilters.exclude[MissingClassProblem](className),
-        ProblemFilters.exclude[MissingTypesProblem](className),
-        excludePackage(className + "$"),
-        ProblemFilters.exclude[MissingClassProblem](className + "$"),
-        ProblemFilters.exclude[MissingTypesProblem](className + "$")
-      )
-    }
-
-    // Exclude a Spark class, that is in the package org.apache.spark
-    def excludeSparkClass(className: String) = {
-      excludeClass("org.apache.spark." + className)
-    }
 
-    // Exclude a Spark package, that is in the package org.apache.spark
-    def excludeSparkPackage(packageName: String) = {
-      excludePackage("org.apache.spark." + packageName)
-    }
 
     val externalExcludeFileClasses = ignoredClasses.flatMap(excludeClass)
 
-    defaultExcludes ++ externalExcludeFileClasses
+    defaultExcludes ++ externalExcludeFileClasses ++ MimaExcludes.excludes
   }
 
   def mimaSettings(sparkHome: File) = mimaDefaultSettings ++ Seq(
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
new file mode 100644
index 0000000000..ecb389de55
--- /dev/null
+++ b/project/MimaExcludes.scala
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import com.typesafe.tools.mima.core._
+import com.typesafe.tools.mima.core.ProblemFilters._
+
+/**
+ * Additional excludes for checking of Spark's binary compatibility.
+ *
+ * The Mima build will automatically exclude @DeveloperApi and @Experimental classes. This acts
+ * as an official audit of cases where we excluded other classes. Please use the narrowest
+ * possible exclude here. MIMA will usually tell you what exclude to use, e.g.:
+ *
+ * ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.rdd.RDD.take")
+ *
+ * It is also possible to exclude Spark classes and packages. This should be used sparingly:
+ *
+ * MimaBuild.excludeSparkClass("graphx.util.collection.GraphXPrimitiveKeyOpenHashMap")
+ */
+object MimaExcludes {
+    val excludes =
+      SparkBuild.SPARK_VERSION match {
+        case v if v.startsWith("1.1") =>
+          Seq()
+        case v if v.startsWith("1.0") =>
+          Seq(
+            MimaBuild.excludeSparkPackage("api.java"),
+            MimaBuild.excludeSparkPackage("mllib"),
+            MimaBuild.excludeSparkPackage("streaming")
+          ) ++
+          MimaBuild.excludeSparkClass("rdd.ClassTags") ++
+          MimaBuild.excludeSparkClass("util.XORShiftRandom") ++
+          MimaBuild.excludeSparkClass("graphx.EdgeRDD") ++
+          MimaBuild.excludeSparkClass("graphx.VertexRDD") ++
+          MimaBuild.excludeSparkClass("graphx.impl.GraphImpl") ++
+          MimaBuild.excludeSparkClass("graphx.impl.RoutingTable") ++
+          MimaBuild.excludeSparkClass("graphx.util.collection.PrimitiveKeyOpenHashMap") ++
+          MimaBuild.excludeSparkClass("graphx.util.collection.GraphXPrimitiveKeyOpenHashMap") ++
+          MimaBuild.excludeSparkClass("mllib.recommendation.MFDataGenerator") ++
+          MimaBuild.excludeSparkClass("mllib.optimization.SquaredGradient") ++
+          MimaBuild.excludeSparkClass("mllib.regression.RidgeRegressionWithSGD") ++
+          MimaBuild.excludeSparkClass("mllib.regression.LassoWithSGD") ++
+          MimaBuild.excludeSparkClass("mllib.regression.LinearRegressionWithSGD")
+        case _ => Seq()
+      }
+}
+