SPARK-1094 Support MiMa for reporting binary compatibility accross versions.

This adds some changes on top of the initial work by @scrapcodes in #20:

The goal here is to do automated checking of Spark commits to determine whether they break binary compatibility.

1. Special case for inner classes of package-private objects.
2. Made tools classes accessible when running `spark-class`.
3. Made some declared types in MLLib more general.
4. Various other improvements to exclude-generation script.
5. In-code documentation.

Author: Patrick Wendell <pwendell@gmail.com>
Author: Prashant Sharma <prashant.s@imaginea.com>
Author: Prashant Sharma <scrapcodes@gmail.com>

Closes #207 from pwendell/mima and squashes the following commits:

22ae267 [Patrick Wendell] New binary changes after upmerge
6c2030d [Patrick Wendell] Merge remote-tracking branch 'apache/master' into mima
3666cf1 [Patrick Wendell] Minor style change
0e0f570 [Patrick Wendell] Small fix and removing directory listings
647c547 [Patrick Wendell] Reveiw feedback.
c39f3b5 [Patrick Wendell] Some enhancements to binary checking.
4c771e0 [Prashant Sharma] Added a tool to generate mima excludes and also adapted build to pick automatically.
b551519 [Prashant Sharma] adding a new exclude after rebasing with master
651844c [Prashant Sharma] Support MiMa for reporting binary compatibility accross versions.

1 files changed, 83 insertions, 0 deletions

diff --git a/project/MimaBuild.scala b/project/MimaBuild.scala
new file mode 100644
index 0000000000..e7c9c47c96
--- /dev/null
+++ b/project/MimaBuild.scala
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import com.typesafe.tools.mima.plugin.MimaKeys.{binaryIssueFilters, previousArtifact}
+import com.typesafe.tools.mima.plugin.MimaPlugin.mimaDefaultSettings
+import sbt._
+
+object MimaBuild {
+
+  def ignoredABIProblems(base: File) = {
+    import com.typesafe.tools.mima.core._
+    import com.typesafe.tools.mima.core.ProblemFilters._
+
+    // Excludes placed here will be used for all Spark versions
+    val defaultExcludes = Seq()
+
+    // Read package-private excludes from file
+    val excludeFilePath = (base.getAbsolutePath + "/.mima-excludes")
+    val excludeFile = file(excludeFilePath) 
+    val packagePrivateList: Seq[String] =
+      if (!excludeFile.exists()) {
+        Seq()
+      } else {
+        IO.read(excludeFile).split("\n")
+      }
+
+    def excludeClass(className: String) = {
+      Seq(
+        excludePackage(className), 
+        ProblemFilters.exclude[MissingClassProblem](className),
+        ProblemFilters.exclude[MissingTypesProblem](className),
+        excludePackage(className + "$"), 
+        ProblemFilters.exclude[MissingClassProblem](className + "$"),
+        ProblemFilters.exclude[MissingTypesProblem](className + "$")
+      )
+    }
+    def excludeSparkClass(className: String) = excludeClass("org.apache.spark." + className)
+
+    val packagePrivateExcludes = packagePrivateList.flatMap(excludeClass)
+
+    /* Excludes specific to a given version of Spark. When comparing the given version against
+       its immediate predecessor, the excludes listed here will be applied. */
+    val versionExcludes =
+      SparkBuild.SPARK_VERSION match {
+        case v if v.startsWith("1.0") =>
+          Seq(
+             excludePackage("org.apache.spark.api.java"),
+             excludePackage("org.apache.spark.streaming.api.java"),
+             excludePackage("org.apache.spark.mllib")
+           ) ++
+           excludeSparkClass("rdd.ClassTags") ++
+           excludeSparkClass("util.XORShiftRandom") ++
+           excludeSparkClass("mllib.recommendation.MFDataGenerator") ++
+           excludeSparkClass("mllib.optimization.SquaredGradient") ++
+           excludeSparkClass("mllib.regression.RidgeRegressionWithSGD") ++
+           excludeSparkClass("mllib.regression.LassoWithSGD") ++
+           excludeSparkClass("mllib.regression.LinearRegressionWithSGD")
+        case _ => Seq()
+      }
+
+    defaultExcludes ++ packagePrivateExcludes ++ versionExcludes
+  }
+
+  def mimaSettings(sparkHome: File) = mimaDefaultSettings ++ Seq(
+    previousArtifact := None,
+    binaryIssueFilters ++= ignoredABIProblems(sparkHome)
+  )
+
+}