SPARK-1094 Support MiMa for reporting binary compatibility accross versions.

This adds some changes on top of the initial work by @scrapcodes in #20:

The goal here is to do automated checking of Spark commits to determine whether they break binary compatibility.

1. Special case for inner classes of package-private objects.
2. Made tools classes accessible when running `spark-class`.
3. Made some declared types in MLLib more general.
4. Various other improvements to exclude-generation script.
5. In-code documentation.

Author: Patrick Wendell <pwendell@gmail.com>
Author: Prashant Sharma <prashant.s@imaginea.com>
Author: Prashant Sharma <scrapcodes@gmail.com>

Closes #207 from pwendell/mima and squashes the following commits:

22ae267 [Patrick Wendell] New binary changes after upmerge
6c2030d [Patrick Wendell] Merge remote-tracking branch 'apache/master' into mima
3666cf1 [Patrick Wendell] Minor style change
0e0f570 [Patrick Wendell] Small fix and removing directory listings
647c547 [Patrick Wendell] Reveiw feedback.
c39f3b5 [Patrick Wendell] Some enhancements to binary checking.
4c771e0 [Prashant Sharma] Added a tool to generate mima excludes and also adapted build to pick automatically.
b551519 [Prashant Sharma] adding a new exclude after rebasing with master
651844c [Prashant Sharma] Support MiMa for reporting binary compatibility accross versions.

1 files changed, 132 insertions, 0 deletions

diff --git a/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala b/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala
new file mode 100644
index 0000000000..5547e9fe58
--- /dev/null
+++ b/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.tools
+
+import java.io.File
+import java.util.jar.JarFile
+
+import scala.collection.mutable
+import scala.collection.JavaConversions._
+import scala.reflect.runtime.universe.runtimeMirror
+
+/**
+ * A tool for generating classes to be excluded during binary checking with MIMA. It is expected
+ * that this tool is run with ./spark-class.
+ *
+ * MIMA itself only supports JVM-level visibility and doesn't account for package-private classes.
+ * This tool looks at all currently package-private classes and generates exclusions for them. Note
+ * that this approach is not sound. It can lead to false positives if we move or rename a previously
+ * package-private class. It can lead to false negatives if someone explicitly makes a class
+ * package-private that wasn't before. This exists only to help catch certain classes of changes
+ * which might be difficult to catch during review.
+ */
+object GenerateMIMAIgnore {
+  private val classLoader = Thread.currentThread().getContextClassLoader
+  private val mirror = runtimeMirror(classLoader)
+
+  private def classesPrivateWithin(packageName: String): Set[String] = {
+
+    val classes = getClasses(packageName, classLoader)
+    val privateClasses = mutable.HashSet[String]()
+
+    def isPackagePrivate(className: String) = {
+      try {
+        /* Couldn't figure out if it's possible to determine a-priori whether a given symbol
+           is a module or class. */
+
+        val privateAsClass = mirror
+          .staticClass(className)
+          .privateWithin
+          .fullName
+          .startsWith(packageName)
+
+        val privateAsModule = mirror
+          .staticModule(className)
+          .privateWithin
+          .fullName
+          .startsWith(packageName)
+
+        privateAsClass || privateAsModule
+      } catch {
+        case _: Throwable => {
+          println("Error determining visibility: " + className)
+          false
+        }
+      }
+    }
+
+    for (className <- classes) {
+      val directlyPrivateSpark = isPackagePrivate(className)
+
+      /* Inner classes defined within a private[spark] class or object are effectively
+         invisible, so we account for them as package private. */
+      val indirectlyPrivateSpark = {
+        val maybeOuter = className.toString.takeWhile(_ != '$')
+        if (maybeOuter != className) {
+          isPackagePrivate(maybeOuter)
+        } else {
+          false
+        }
+      }
+      if (directlyPrivateSpark || indirectlyPrivateSpark) privateClasses += className
+    }
+    privateClasses.flatMap(c => Seq(c, c.replace("$", "#"))).toSet
+  }
+
+  def main(args: Array[String]) {
+    scala.tools.nsc.io.File(".mima-excludes").
+      writeAll(classesPrivateWithin("org.apache.spark").mkString("\n"))
+    println("Created : .mima-excludes in current directory.")
+  }
+
+
+  private def shouldExclude(name: String) = {
+    // Heuristic to remove JVM classes that do not correspond to user-facing classes in Scala
+    name.contains("anon") ||
+    name.endsWith("$class") ||
+    name.contains("$sp")
+  }
+
+  /**
+   * Scans all classes accessible from the context class loader which belong to the given package
+   * and subpackages both from directories and jars present on the classpath.
+   */
+  private def getClasses(packageName: String,
+      classLoader: ClassLoader = Thread.currentThread().getContextClassLoader): Set[String] = {
+    val path = packageName.replace('.', '/')
+    val resources = classLoader.getResources(path)
+
+    val jars = resources.filter(x => x.getProtocol == "jar")
+      .map(_.getFile.split(":")(1).split("!")(0)).toSeq
+
+    jars.flatMap(getClassesFromJar(_, path))
+      .map(_.getName)
+      .filterNot(shouldExclude).toSet
+  }
+
+  /**
+   * Get all classes in a package from a jar file.
+   */
+  private def getClassesFromJar(jarPath: String, packageName: String) = {
+    val jar = new JarFile(new File(jarPath))
+    val enums = jar.entries().map(_.getName).filter(_.startsWith(packageName))
+    val classes = for (entry <- enums if entry.endsWith(".class"))
+      yield Class.forName(entry.replace('/', '.').stripSuffix(".class"))
+    classes
+  }
+}