diff options
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | .rat-excludes | 1 | ||||
| -rwxr-xr-x | dev/mima | 34 | ||||
| -rwxr-xr-x | dev/run-tests | 3 | ||||
| -rw-r--r-- | project/MimaBuild.scala | 48 | ||||
| -rw-r--r-- | project/MimaExcludes.scala | 61 | ||||
| -rw-r--r-- | tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala | 4 |
7 files changed, 124 insertions, 29 deletions
diff --git a/.gitignore b/.gitignore index ad72588b47..b54a3058de 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ sbt/*.jar .settings .cache -.mima-excludes +.generated-mima-excludes /build/ work/ out/ diff --git a/.rat-excludes b/.rat-excludes index 15589702c5..52b2dfac5c 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -3,6 +3,7 @@ target .project .classpath .mima-excludes +.generated-mima-excludes .rat-excludes .*md derby.log diff --git a/dev/mima b/dev/mima new file mode 100755 index 0000000000..d409999025 --- /dev/null +++ b/dev/mima @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -o pipefail + +# Go to the Spark project root directory +FWDIR="$(cd `dirname $0`/..; pwd)" +cd $FWDIR + +./bin/spark-class org.apache.spark.tools.GenerateMIMAIgnore +echo -e "q\n" | sbt/sbt mima-report-binary-issues | grep -v -e "info.*Resolving" +ret_val=$? + +if [ $ret_val != 0 ]; then + echo "NOTE: Exceptions to binary compatibility can be added in project/MimaExcludes.scala" +fi + +exit $ret_val diff --git a/dev/run-tests b/dev/run-tests index 6043f859ae..93d6692f83 100755 --- a/dev/run-tests +++ b/dev/run-tests @@ -81,5 +81,4 @@ fi echo "=========================================================================" echo "Detecting binary incompatibilites with MiMa" echo "=========================================================================" -./bin/spark-class org.apache.spark.tools.GenerateMIMAIgnore -echo -e "q\n" | sbt/sbt mima-report-binary-issues | grep -v -e "info.*Resolving" +dev/mima diff --git a/project/MimaBuild.scala b/project/MimaBuild.scala index 182ca7615d..1477809943 100644 --- a/project/MimaBuild.scala +++ b/project/MimaBuild.scala @@ -15,21 +15,41 @@ * limitations under the License. */ +import com.typesafe.tools.mima.core.{MissingTypesProblem, MissingClassProblem, ProblemFilters} +import com.typesafe.tools.mima.core.ProblemFilters._ import com.typesafe.tools.mima.plugin.MimaKeys.{binaryIssueFilters, previousArtifact} import com.typesafe.tools.mima.plugin.MimaPlugin.mimaDefaultSettings import sbt._ object MimaBuild { + // Exclude a single class and its corresponding object + def excludeClass(className: String) = { + Seq( + excludePackage(className), + ProblemFilters.exclude[MissingClassProblem](className), + ProblemFilters.exclude[MissingTypesProblem](className), + excludePackage(className + "$"), + ProblemFilters.exclude[MissingClassProblem](className + "$"), + ProblemFilters.exclude[MissingTypesProblem](className + "$") + ) + } + // Exclude a Spark class, that is in the package org.apache.spark + def excludeSparkClass(className: String) = { + excludeClass("org.apache.spark." + className) + } + + // Exclude a Spark package, that is in the package org.apache.spark + def excludeSparkPackage(packageName: String) = { + excludePackage("org.apache.spark." + packageName) + } def ignoredABIProblems(base: File) = { - import com.typesafe.tools.mima.core._ - import com.typesafe.tools.mima.core.ProblemFilters._ // Excludes placed here will be used for all Spark versions val defaultExcludes = Seq() // Read package-private excludes from file - val excludeFilePath = (base.getAbsolutePath + "/.mima-excludes") + val excludeFilePath = (base.getAbsolutePath + "/.generated-mima-excludes") val excludeFile = file(excludeFilePath) val ignoredClasses: Seq[String] = if (!excludeFile.exists()) { @@ -38,31 +58,11 @@ object MimaBuild { IO.read(excludeFile).split("\n") } - // Exclude a single class and its corresponding object - def excludeClass(className: String) = { - Seq( - excludePackage(className), - ProblemFilters.exclude[MissingClassProblem](className), - ProblemFilters.exclude[MissingTypesProblem](className), - excludePackage(className + "$"), - ProblemFilters.exclude[MissingClassProblem](className + "$"), - ProblemFilters.exclude[MissingTypesProblem](className + "$") - ) - } - - // Exclude a Spark class, that is in the package org.apache.spark - def excludeSparkClass(className: String) = { - excludeClass("org.apache.spark." + className) - } - // Exclude a Spark package, that is in the package org.apache.spark - def excludeSparkPackage(packageName: String) = { - excludePackage("org.apache.spark." + packageName) - } val externalExcludeFileClasses = ignoredClasses.flatMap(excludeClass) - defaultExcludes ++ externalExcludeFileClasses + defaultExcludes ++ externalExcludeFileClasses ++ MimaExcludes.excludes } def mimaSettings(sparkHome: File) = mimaDefaultSettings ++ Seq( diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala new file mode 100644 index 0000000000..ecb389de55 --- /dev/null +++ b/project/MimaExcludes.scala @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.typesafe.tools.mima.core._ +import com.typesafe.tools.mima.core.ProblemFilters._ + +/** + * Additional excludes for checking of Spark's binary compatibility. + * + * The Mima build will automatically exclude @DeveloperApi and @Experimental classes. This acts + * as an official audit of cases where we excluded other classes. Please use the narrowest + * possible exclude here. MIMA will usually tell you what exclude to use, e.g.: + * + * ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.rdd.RDD.take") + * + * It is also possible to exclude Spark classes and packages. This should be used sparingly: + * + * MimaBuild.excludeSparkClass("graphx.util.collection.GraphXPrimitiveKeyOpenHashMap") + */ +object MimaExcludes { + val excludes = + SparkBuild.SPARK_VERSION match { + case v if v.startsWith("1.1") => + Seq() + case v if v.startsWith("1.0") => + Seq( + MimaBuild.excludeSparkPackage("api.java"), + MimaBuild.excludeSparkPackage("mllib"), + MimaBuild.excludeSparkPackage("streaming") + ) ++ + MimaBuild.excludeSparkClass("rdd.ClassTags") ++ + MimaBuild.excludeSparkClass("util.XORShiftRandom") ++ + MimaBuild.excludeSparkClass("graphx.EdgeRDD") ++ + MimaBuild.excludeSparkClass("graphx.VertexRDD") ++ + MimaBuild.excludeSparkClass("graphx.impl.GraphImpl") ++ + MimaBuild.excludeSparkClass("graphx.impl.RoutingTable") ++ + MimaBuild.excludeSparkClass("graphx.util.collection.PrimitiveKeyOpenHashMap") ++ + MimaBuild.excludeSparkClass("graphx.util.collection.GraphXPrimitiveKeyOpenHashMap") ++ + MimaBuild.excludeSparkClass("mllib.recommendation.MFDataGenerator") ++ + MimaBuild.excludeSparkClass("mllib.optimization.SquaredGradient") ++ + MimaBuild.excludeSparkClass("mllib.regression.RidgeRegressionWithSGD") ++ + MimaBuild.excludeSparkClass("mllib.regression.LassoWithSGD") ++ + MimaBuild.excludeSparkClass("mllib.regression.LinearRegressionWithSGD") + case _ => Seq() + } +} + diff --git a/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala b/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala index 011db50b7d..6a261e19a3 100644 --- a/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala +++ b/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala @@ -105,9 +105,9 @@ object GenerateMIMAIgnore { } def main(args: Array[String]) { - scala.tools.nsc.io.File(".mima-excludes"). + scala.tools.nsc.io.File(".generated-mima-excludes"). writeAll(classesPrivateWithin("org.apache.spark").mkString("\n")) - println("Created : .mima-excludes in current directory.") + println("Created : .generated-mima-excludes in current directory.") } |