[SPARK-16685] Remove audit-release scripts.

## What changes were proposed in this pull request?
This patch removes dev/audit-release. It was initially created to do basic release auditing. They have been unused by for the last one year+.

## How was this patch tested?
N/A

Author: Reynold Xin <rxin@databricks.com>

Closes #14342 from rxin/SPARK-16685.

24 files changed, 0 insertions, 1020 deletions

diff --git a/dev/audit-release/.gitignore b/dev/audit-release/.gitignore
deleted file mode 100644
index 7e057a92b3..0000000000
--- a/dev/audit-release/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-project/
-spark_audit*
diff --git a/dev/audit-release/README.md b/dev/audit-release/README.md
deleted file mode 100644
index 37b2a0afb7..0000000000
--- a/dev/audit-release/README.md
+++ /dev/null
@@ -1,12 +0,0 @@
-Test Application Builds
-=======================
-
-This directory includes test applications which are built when auditing releases. You can run them locally by setting appropriate environment variables.
-
-```
-$ cd sbt_app_core
-$ SCALA_VERSION=2.11.7 \
-  SPARK_VERSION=1.0.0-SNAPSHOT \
-  SPARK_RELEASE_REPOSITORY=file:///home/patrick/.ivy2/local \
-  sbt run
-```
diff --git a/dev/audit-release/audit_release.py b/dev/audit-release/audit_release.py
deleted file mode 100755
index b28e7a427b..0000000000
--- a/dev/audit-release/audit_release.py
+++ /dev/null
@@ -1,236 +0,0 @@
-#!/usr/bin/python
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Audits binary and maven artifacts for a Spark release.
-# Requires GPG and Maven.
-# usage:
-#   python audit_release.py
-
-import os
-import re
-import shutil
-import subprocess
-import sys
-import time
-import urllib2
-
-# Note: The following variables must be set before use!
-RELEASE_URL = "http://people.apache.org/~andrewor14/spark-1.1.1-rc1/"
-RELEASE_KEY = "XXXXXXXX" # Your 8-digit hex
-RELEASE_REPOSITORY = "https://repository.apache.org/content/repositories/orgapachespark-1033"
-RELEASE_VERSION = "1.1.1"
-SCALA_VERSION = "2.11.7"
-SCALA_BINARY_VERSION = "2.11"
-
-# Do not set these
-LOG_FILE_NAME = "spark_audit_%s" % time.strftime("%h_%m_%Y_%I_%M_%S")
-LOG_FILE = open(LOG_FILE_NAME, 'w')
-WORK_DIR = "/tmp/audit_%s" % int(time.time())
-MAVEN_CMD = "mvn"
-GPG_CMD = "gpg"
-SBT_CMD = "sbt -Dsbt.log.noformat=true"
-
-# Track failures to print them at the end
-failures = []
-
-# Log a message. Use sparingly because this flushes every write.
-def log(msg):
-    LOG_FILE.write(msg + "\n")
-    LOG_FILE.flush()
-
-def log_and_print(msg):
-    print msg
-    log(msg)
-
-# Prompt the user to delete the scratch directory used
-def clean_work_files():
-    response = raw_input("OK to delete scratch directory '%s'? (y/N) " % WORK_DIR)
-    if response == "y":
-        shutil.rmtree(WORK_DIR)
-
-# Run the given command and log its output to the log file
-def run_cmd(cmd, exit_on_failure=True):
-    log("Running command: %s" % cmd)
-    ret = subprocess.call(cmd, shell=True, stdout=LOG_FILE, stderr=LOG_FILE)
-    if ret != 0 and exit_on_failure:
-        log_and_print("Command failed: %s" % cmd)
-        clean_work_files()
-        sys.exit(-1)
-    return ret
-
-def run_cmd_with_output(cmd):
-    log_and_print("Running command: %s" % cmd)
-    return subprocess.check_output(cmd, shell=True, stderr=LOG_FILE)
-
-# Test if the given condition is successful
-# If so, print the pass message; otherwise print the failure message
-def test(cond, msg):
-    return passed(msg) if cond else failed(msg)
-
-def passed(msg):
-    log_and_print("[PASSED] %s" % msg)
-
-def failed(msg):
-    failures.append(msg)
-    log_and_print("[**FAILED**] %s" % msg)
-
-def get_url(url):
-    return urllib2.urlopen(url).read()
-
-# If the path exists, prompt the user to delete it
-# If the resource is not deleted, abort
-def ensure_path_not_present(path):
-    full_path = os.path.expanduser(path)
-    if os.path.exists(full_path):
-        print "Found %s locally." % full_path
-        response = raw_input("This can interfere with testing published artifacts. OK to delete? (y/N) ")
-        if response == "y":
-            shutil.rmtree(full_path)
-        else:
-            print "Abort."
-            sys.exit(-1)
-
-log_and_print("|-------- Starting Spark audit tests for release %s --------|" % RELEASE_VERSION)
-log_and_print("Log output can be found in %s" % LOG_FILE_NAME)
-
-original_dir = os.getcwd()
-
-# For each of these modules, we'll test an 'empty' application in sbt and
-# maven that links against them. This will catch issues with messed up
-# dependencies within those projects.
-modules = [
-    "spark-core", "spark-mllib", "spark-streaming", "spark-repl",
-    "spark-graphx", "spark-streaming-flume", "spark-streaming-kafka-0-8",
-    "spark-catalyst", "spark-sql", "spark-hive", "spark-streaming-kinesis-asl"
-]
-modules = map(lambda m: "%s_%s" % (m, SCALA_BINARY_VERSION), modules)
-
-# Check for directories that might interfere with tests
-local_ivy_spark = "~/.ivy2/local/org.apache.spark"
-cache_ivy_spark = "~/.ivy2/cache/org.apache.spark"
-local_maven_kafka = "~/.m2/repository/org/apache/kafka"
-local_maven_kafka = "~/.m2/repository/org/apache/spark"
-map(ensure_path_not_present, [local_ivy_spark, cache_ivy_spark, local_maven_kafka])
-
-# SBT build tests
-log_and_print("==== Building SBT modules ====")
-os.chdir("blank_sbt_build")
-os.environ["SPARK_VERSION"] = RELEASE_VERSION
-os.environ["SCALA_VERSION"] = SCALA_VERSION
-os.environ["SPARK_RELEASE_REPOSITORY"] = RELEASE_REPOSITORY
-os.environ["SPARK_AUDIT_MASTER"] = "local"
-for module in modules:
-    log("==== Building module %s in SBT ====" % module)
-    os.environ["SPARK_MODULE"] = module
-    ret = run_cmd("%s clean update" % SBT_CMD, exit_on_failure=False)
-    test(ret == 0, "SBT build against '%s' module" % module)
-os.chdir(original_dir)
-
-# SBT application tests
-log_and_print("==== Building SBT applications ====")
-for app in ["sbt_app_core", "sbt_app_graphx", "sbt_app_streaming", "sbt_app_sql", "sbt_app_hive", "sbt_app_kinesis"]:
-    log("==== Building application %s in SBT ====" % app)
-    os.chdir(app)
-    ret = run_cmd("%s clean run" % SBT_CMD, exit_on_failure=False)
-    test(ret == 0, "SBT application (%s)" % app)
-    os.chdir(original_dir)
-
-# Maven build tests
-os.chdir("blank_maven_build")
-log_and_print("==== Building Maven modules ====")
-for module in modules:
-    log("==== Building module %s in maven ====" % module)
-    cmd = ('%s --update-snapshots -Dspark.release.repository="%s" -Dspark.version="%s" '
-           '-Dspark.module="%s" clean compile' %
-           (MAVEN_CMD, RELEASE_REPOSITORY, RELEASE_VERSION, module))
-    ret = run_cmd(cmd, exit_on_failure=False)
-    test(ret == 0, "maven build against '%s' module" % module)
-os.chdir(original_dir)
-
-# Maven application tests
-log_and_print("==== Building Maven applications ====")
-os.chdir("maven_app_core")
-mvn_exec_cmd = ('%s --update-snapshots -Dspark.release.repository="%s" -Dspark.version="%s" '
-                '-Dscala.binary.version="%s" clean compile '
-                'exec:java -Dexec.mainClass="SimpleApp"' %
-                (MAVEN_CMD, RELEASE_REPOSITORY, RELEASE_VERSION, SCALA_BINARY_VERSION))
-ret = run_cmd(mvn_exec_cmd, exit_on_failure=False)
-test(ret == 0, "maven application (core)")
-os.chdir(original_dir)
-
-# Binary artifact tests
-if os.path.exists(WORK_DIR):
-    print "Working directory '%s' already exists" % WORK_DIR
-    sys.exit(-1)
-os.mkdir(WORK_DIR)
-os.chdir(WORK_DIR)
-
-index_page = get_url(RELEASE_URL)
-artifact_regex = r = re.compile("<a href=\"(.*.tgz)\">")
-artifacts = r.findall(index_page)
-
-# Verify artifact integrity
-for artifact in artifacts:
-    log_and_print("==== Verifying download integrity for artifact: %s ====" % artifact)
-
-    artifact_url = "%s/%s" % (RELEASE_URL, artifact)
-    key_file = "%s.asc" % artifact
-    run_cmd("wget %s" % artifact_url)
-    run_cmd("wget %s/%s" % (RELEASE_URL, key_file))
-    run_cmd("wget %s%s" % (artifact_url, ".sha"))
-
-    # Verify signature
-    run_cmd("%s --keyserver pgp.mit.edu --recv-key %s" % (GPG_CMD, RELEASE_KEY))
-    run_cmd("%s %s" % (GPG_CMD, key_file))
-    passed("Artifact signature verified.")
-
-    # Verify md5
-    my_md5 = run_cmd_with_output("%s --print-md MD5 %s" % (GPG_CMD, artifact)).strip()
-    release_md5 = get_url("%s.md5" % artifact_url).strip()
-    test(my_md5 == release_md5, "Artifact MD5 verified.")
-
-    # Verify sha
-    my_sha = run_cmd_with_output("%s --print-md SHA512 %s" % (GPG_CMD, artifact)).strip()
-    release_sha = get_url("%s.sha" % artifact_url).strip()
-    test(my_sha == release_sha, "Artifact SHA verified.")
-
-    # Verify Apache required files
-    dir_name = artifact.replace(".tgz", "")
-    run_cmd("tar xvzf %s" % artifact)
-    base_files = os.listdir(dir_name)
-    test("CHANGES.txt" in base_files, "Tarball contains CHANGES.txt file")
-    test("NOTICE" in base_files, "Tarball contains NOTICE file")
-    test("LICENSE" in base_files, "Tarball contains LICENSE file")
-
-    os.chdir(WORK_DIR)
-
-# Report result
-log_and_print("\n")
-if len(failures) == 0:
-    log_and_print("*** ALL TESTS PASSED ***")
-else:
-    log_and_print("XXXXX SOME TESTS DID NOT PASS XXXXX")
-    for f in failures:
-        log_and_print("  %s" % f)
-os.chdir(original_dir)
-
-# Clean up
-clean_work_files()
-
-log_and_print("|-------- Spark release audit complete --------|")
diff --git a/dev/audit-release/blank_maven_build/pom.xml b/dev/audit-release/blank_maven_build/pom.xml
deleted file mode 100644
index 02dd9046c9..0000000000
--- a/dev/audit-release/blank_maven_build/pom.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one or more
-  ~ contributor license agreements.  See the NOTICE file distributed with
-  ~ this work for additional information regarding copyright ownership.
-  ~ The ASF licenses this file to You under the Apache License, Version 2.0
-  ~ (the "License"); you may not use this file except in compliance with
-  ~ the License.  You may obtain a copy of the License at
-  ~
-  ~    http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<project>
-  <groupId>spark.audit</groupId>
-  <artifactId>spark-audit</artifactId>
-  <modelVersion>4.0.0</modelVersion>
-  <name>Spark Release Auditor</name>
-  <packaging>jar</packaging>
-  <version>1.0</version>
-  <repositories>
-    <repository>
-      <id>Spray.cc repository</id>
-      <url>http://repo.spray.cc</url>
-    </repository>
-    <repository>
-      <id>Spark Staging Repo</id>
-      <url>${spark.release.repository}</url>
-    </repository>
-  </repositories>
-  <dependencies>
-    <dependency>
-      <groupId>org.apache.spark</groupId>
-      <artifactId>${spark.module}</artifactId>
-      <version>${spark.version}</version>
-    </dependency>
-  </dependencies>
-</project>
diff --git a/dev/audit-release/blank_sbt_build/build.sbt b/dev/audit-release/blank_sbt_build/build.sbt
deleted file mode 100644
index 62815542e5..0000000000
--- a/dev/audit-release/blank_sbt_build/build.sbt
+++ /dev/null
@@ -1,30 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements.  See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License.  You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-name := "Spark Release Auditor"
-
-version := "1.0"
-
-scalaVersion := System.getenv.get("SCALA_VERSION")
-
-libraryDependencies += "org.apache.spark" % System.getenv.get("SPARK_MODULE") % System.getenv.get("SPARK_VERSION")
-
-resolvers ++= Seq(
-  "Spark Release Repository" at System.getenv.get("SPARK_RELEASE_REPOSITORY"),
-  "Eclipse Paho Repository" at "https://repo.eclipse.org/content/repositories/paho-releases/",
-  "Maven Repository" at "http://repo1.maven.org/maven2/",
-  "Spray Repository" at "http://repo.spray.cc/")
diff --git a/dev/audit-release/maven_app_core/input.txt b/dev/audit-release/maven_app_core/input.txt
deleted file mode 100644
index 837b6f85ae..0000000000
--- a/dev/audit-release/maven_app_core/input.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-a
-b
-c
-d
-a
-b
-c
-d
diff --git a/dev/audit-release/maven_app_core/pom.xml b/dev/audit-release/maven_app_core/pom.xml
deleted file mode 100644
index b516396825..0000000000
--- a/dev/audit-release/maven_app_core/pom.xml
+++ /dev/null
@@ -1,52 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one or more
-  ~ contributor license agreements.  See the NOTICE file distributed with
-  ~ this work for additional information regarding copyright ownership.
-  ~ The ASF licenses this file to You under the Apache License, Version 2.0
-  ~ (the "License"); you may not use this file except in compliance with
-  ~ the License.  You may obtain a copy of the License at
-  ~
-  ~    http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<project>
-  <groupId>spark.audit</groupId>
-  <artifactId>spark-audit</artifactId>
-  <modelVersion>4.0.0</modelVersion>
-  <name>Simple Project</name>
-  <packaging>jar</packaging>
-  <version>1.0</version>
-  <repositories>
-    <repository>
-      <id>Spray.cc repository</id>
-      <url>http://repo.spray.cc</url>
-    </repository>
-    <repository>
-      <id>Spark Staging Repo</id>
-      <url>${spark.release.repository}</url>
-    </repository>
-  </repositories>
-  <dependencies>
-    <dependency> <!-- Spark dependency -->
-      <groupId>org.apache.spark</groupId>
-      <artifactId>spark-core_${scala.binary.version}</artifactId>
-      <version>${spark.version}</version>
-    </dependency>
-  </dependencies>
-  <!-- Makes sure we get a fairly recent compiler plugin. -->
-  <build>
-    <plugins>
-      <plugin>
-        <artifactId>maven-compiler-plugin</artifactId>
-        <version>3.1</version>
-      </plugin>
-    </plugins>
-  </build>
-</project>
diff --git a/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java b/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
deleted file mode 100644
index 5217689e7c..0000000000
--- a/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.spark.api.java.*;
-import org.apache.spark.api.java.function.Function;
-
-public class SimpleApp {
-  public static void main(String[] args) {
-    String logFile = "input.txt";
-    JavaSparkContext sc = new JavaSparkContext("local", "Simple App");
-    JavaRDD<String> logData = sc.textFile(logFile).cache();
-
-    long numAs = logData.filter(new Function<String, Boolean>() {
-      public Boolean call(String s) { return s.contains("a"); }
-    }).count();
-
-    long numBs = logData.filter(new Function<String, Boolean>() {
-      public Boolean call(String s) { return s.contains("b"); }
-    }).count();
-
-   if (numAs != 2 || numBs != 2) {
-     System.out.println("Failed to parse log files with Spark");
-     System.exit(-1);
-   }
-   System.out.println("Test succeeded");
-   sc.stop();
-  }
-}
diff --git a/dev/audit-release/sbt_app_core/build.sbt b/dev/audit-release/sbt_app_core/build.sbt
deleted file mode 100644
index 291b1d6440..0000000000
--- a/dev/audit-release/sbt_app_core/build.sbt
+++ /dev/null
@@ -1,28 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements.  See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License.  You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-name := "Simple Project"
-
-version := "1.0"
-
-scalaVersion := System.getenv.get("SCALA_VERSION")
-
-libraryDependencies += "org.apache.spark" %% "spark-core" % System.getenv.get("SPARK_VERSION")
-
-resolvers ++= Seq(
-  "Spark Release Repository" at System.getenv.get("SPARK_RELEASE_REPOSITORY"),
-  "Spray Repository" at "http://repo.spray.cc/")
diff --git a/dev/audit-release/sbt_app_core/input.txt b/dev/audit-release/sbt_app_core/input.txt
deleted file mode 100644
index 837b6f85ae..0000000000
--- a/dev/audit-release/sbt_app_core/input.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-a
-b
-c
-d
-a
-b
-c
-d
diff --git a/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala b/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala
deleted file mode 100644
index 61d91c70e9..0000000000
--- a/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package main.scala
-
-import scala.util.Try
-
-import org.apache.spark.SparkConf
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-
-object SimpleApp {
-  def main(args: Array[String]) {
-    val conf = sys.env.get("SPARK_AUDIT_MASTER") match {
-      case Some(master) => new SparkConf().setAppName("Simple Spark App").setMaster(master)
-      case None => new SparkConf().setAppName("Simple Spark App")
-    }
-    val logFile = "input.txt"
-    val sc = new SparkContext(conf)
-    val logData = sc.textFile(logFile, 2).cache()
-    val numAs = logData.filter(line => line.contains("a")).count()
-    val numBs = logData.filter(line => line.contains("b")).count()
-    if (numAs != 2 || numBs != 2) {
-      println("Failed to parse log files with Spark")
-      System.exit(-1)
-    }
-
-    // Regression test for SPARK-1167: Remove metrics-ganglia from default build due to LGPL issue
-    val foundConsole = Try(Class.forName("org.apache.spark.metrics.sink.ConsoleSink")).isSuccess
-    val foundGanglia = Try(Class.forName("org.apache.spark.metrics.sink.GangliaSink")).isSuccess
-    if (!foundConsole) {
-      println("Console sink not loaded via spark-core")
-      System.exit(-1)
-    }
-    if (foundGanglia) {
-      println("Ganglia sink was loaded via spark-core")
-      System.exit(-1)
-    }
-
-    // Remove kinesis from default build due to ASL license issue
-    val foundKinesis = Try(Class.forName("org.apache.spark.streaming.kinesis.KinesisUtils")).isSuccess
-    if (foundKinesis) {
-      println("Kinesis was loaded via spark-core")
-      System.exit(-1)
-    }
-  }
-}
-// scalastyle:on println
diff --git a/dev/audit-release/sbt_app_ganglia/build.sbt b/dev/audit-release/sbt_app_ganglia/build.sbt
deleted file mode 100644
index 6d9474acf5..0000000000
--- a/dev/audit-release/sbt_app_ganglia/build.sbt
+++ /dev/null
@@ -1,30 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements.  See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License.  You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-name := "Ganglia Test"
-
-version := "1.0"
-
-scalaVersion := System.getenv.get("SCALA_VERSION")
-
-libraryDependencies += "org.apache.spark" %% "spark-core" % System.getenv.get("SPARK_VERSION")
-
-libraryDependencies += "org.apache.spark" %% "spark-ganglia-lgpl" % System.getenv.get("SPARK_VERSION")
-
-resolvers ++= Seq(
-  "Spark Release Repository" at System.getenv.get("SPARK_RELEASE_REPOSITORY"),
-  "Spray Repository" at "http://repo.spray.cc/")
diff --git a/dev/audit-release/sbt_app_ganglia/src/main/scala/SparkApp.scala b/dev/audit-release/sbt_app_ganglia/src/main/scala/SparkApp.scala
deleted file mode 100644
index 9f7ae75d0b..0000000000
--- a/dev/audit-release/sbt_app_ganglia/src/main/scala/SparkApp.scala
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package main.scala
-
-import scala.util.Try
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-
-object SimpleApp {
-  def main(args: Array[String]) {
-    // Regression test for SPARK-1167: Remove metrics-ganglia from default build due to LGPL issue
-    val foundConsole = Try(Class.forName("org.apache.spark.metrics.sink.ConsoleSink")).isSuccess
-    val foundGanglia = Try(Class.forName("org.apache.spark.metrics.sink.GangliaSink")).isSuccess
-    if (!foundConsole) {
-      println("Console sink not loaded via spark-core")
-      System.exit(-1)
-    }
-    if (!foundGanglia) {
-      println("Ganglia sink not loaded via spark-ganglia-lgpl")
-      System.exit(-1)
-    }
-  }
-}
-// scalastyle:on println
diff --git a/dev/audit-release/sbt_app_graphx/build.sbt b/dev/audit-release/sbt_app_graphx/build.sbt
deleted file mode 100644
index dd11245e67..0000000000
--- a/dev/audit-release/sbt_app_graphx/build.sbt
+++ /dev/null
@@ -1,28 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements.  See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License.  You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-name := "Simple Project"
-
-version := "1.0"
-
-scalaVersion := System.getenv.get("SCALA_VERSION")
-
-libraryDependencies += "org.apache.spark" %% "spark-graphx" % System.getenv.get("SPARK_VERSION")
-
-resolvers ++= Seq(
-  "Spark Release Repository" at System.getenv.get("SPARK_RELEASE_REPOSITORY"),
-  "Spray Repository" at "http://repo.spray.cc/")
diff --git a/dev/audit-release/sbt_app_graphx/src/main/scala/GraphxApp.scala b/dev/audit-release/sbt_app_graphx/src/main/scala/GraphxApp.scala
deleted file mode 100644
index 2f0b6ef9a5..0000000000
--- a/dev/audit-release/sbt_app_graphx/src/main/scala/GraphxApp.scala
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package main.scala
-
-import org.apache.spark.{SparkContext, SparkConf}
-import org.apache.spark.SparkContext._
-import org.apache.spark.graphx._
-import org.apache.spark.rdd.RDD
-
-object GraphXApp {
-  def main(args: Array[String]) {
-    val conf = sys.env.get("SPARK_AUDIT_MASTER") match {
-      case Some(master) => new SparkConf().setAppName("Simple GraphX App").setMaster(master)
-      case None => new SparkConf().setAppName("Simple Graphx App")
-    }
-    val sc = new SparkContext(conf)
-    SparkContext.jarOfClass(this.getClass).foreach(sc.addJar)
-
-    val users: RDD[(VertexId, (String, String))] =
-      sc.parallelize(Array((3L, ("rxin", "student")), (7L, ("jgonzal", "postdoc")),
-                           (5L, ("franklin", "prof")), (2L, ("istoica", "prof")),
-                           (4L, ("peter", "student"))))
-    val relationships: RDD[Edge[String]] =
-      sc.parallelize(Array(Edge(3L, 7L, "collab"),    Edge(5L, 3L, "advisor"),
-                           Edge(2L, 5L, "colleague"), Edge(5L, 7L, "pi"),
-                           Edge(4L, 0L, "student"),   Edge(5L, 0L, "colleague")))
-    val defaultUser = ("John Doe", "Missing")
-    val graph = Graph(users, relationships, defaultUser)
-    // Notice that there is a user 0 (for which we have no information) connected to users
-    // 4 (peter) and 5 (franklin).
-    val triplets = graph.triplets.map(e => (e.srcAttr._1, e.dstAttr._1)).collect
-    if (!triplets.exists(_ == ("peter", "John Doe"))) {
-      println("Failed to run GraphX")
-      System.exit(-1)
-    }
-    println("Test succeeded")
-  }
-}
-// scalastyle:on println
diff --git a/dev/audit-release/sbt_app_hive/build.sbt b/dev/audit-release/sbt_app_hive/build.sbt
deleted file mode 100644
index c8824f2b15..0000000000
--- a/dev/audit-release/sbt_app_hive/build.sbt
+++ /dev/null
@@ -1,29 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements.  See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License.  You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-name := "Simple Project"
-
-version := "1.0"
-
-scalaVersion := System.getenv.get("SCALA_VERSION")
-
-libraryDependencies += "org.apache.spark" %% "spark-hive" % System.getenv.get("SPARK_VERSION")
-
-resolvers ++= Seq(
-  "Spark Release Repository" at System.getenv.get("SPARK_RELEASE_REPOSITORY"),
-  "Maven Repository" at "http://repo1.maven.org/maven2/",
-  "Spray Repository" at "http://repo.spray.cc/")
diff --git a/dev/audit-release/sbt_app_hive/data.txt b/dev/audit-release/sbt_app_hive/data.txt
deleted file mode 100644
index 0229e67f51..0000000000
--- a/dev/audit-release/sbt_app_hive/data.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-0
val_0
-1
val_1
-2
val_2
-3
val_3
-4
val_4
-5
val_5
-6
val_6
-7
val_7
-9
val_9
diff --git a/dev/audit-release/sbt_app_hive/src/main/scala/HiveApp.scala b/dev/audit-release/sbt_app_hive/src/main/scala/HiveApp.scala
deleted file mode 100644
index 8cbfb9cd41..0000000000
--- a/dev/audit-release/sbt_app_hive/src/main/scala/HiveApp.scala
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package main.scala
-
-import scala.collection.mutable.{ListBuffer, Queue}
-
-import org.apache.spark.{SparkConf, SparkContext, SparkSession}
-import org.apache.spark.rdd.RDD
-
-case class Person(name: String, age: Int)
-
-object SparkSqlExample {
-
-  def main(args: Array[String]) {
-    val conf = sys.env.get("SPARK_AUDIT_MASTER") match {
-      case Some(master) => new SparkConf().setAppName("Simple Sql App").setMaster(master)
-      case None => new SparkConf().setAppName("Simple Sql App")
-    }
-    val sc = new SparkContext(conf)
-    val sparkSession = SparkSession.builder
-      .enableHiveSupport()
-      .getOrCreate()
-
-    import sparkSession._
-    sql("DROP TABLE IF EXISTS src")
-    sql("CREATE TABLE IF NOT EXISTS src (key INT, value STRING)")
-    sql("LOAD DATA LOCAL INPATH 'data.txt' INTO TABLE src")
-    val results = sql("FROM src SELECT key, value WHERE key >= 0 AND KEY < 5").collect()
-    results.foreach(println)
-
-    def test(f: => Boolean, failureMsg: String) = {
-      if (!f) {
-        println(failureMsg)
-        System.exit(-1)
-      }
-    }
-
-    test(results.size == 5, "Unexpected number of selected elements: " + results)
-    println("Test succeeded")
-    sc.stop()
-  }
-}
-// scalastyle:on println
diff --git a/dev/audit-release/sbt_app_kinesis/build.sbt b/dev/audit-release/sbt_app_kinesis/build.sbt
deleted file mode 100644
index 981bc7957b..0000000000
--- a/dev/audit-release/sbt_app_kinesis/build.sbt
+++ /dev/null
@@ -1,28 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements.  See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License.  You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-name := "Kinesis Test"
-
-version := "1.0"
-
-scalaVersion := System.getenv.get("SCALA_VERSION")
-
-libraryDependencies += "org.apache.spark" %% "spark-streaming-kinesis-asl" % System.getenv.get("SPARK_VERSION")
-
-resolvers ++= Seq(
-  "Spark Release Repository" at System.getenv.get("SPARK_RELEASE_REPOSITORY"),
-  "Spray Repository" at "http://repo.spray.cc/")
diff --git a/dev/audit-release/sbt_app_kinesis/src/main/scala/SparkApp.scala b/dev/audit-release/sbt_app_kinesis/src/main/scala/SparkApp.scala
deleted file mode 100644
index adc25b57d6..0000000000
--- a/dev/audit-release/sbt_app_kinesis/src/main/scala/SparkApp.scala
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package main.scala
-
-import scala.util.Try
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-
-object SimpleApp {
-  def main(args: Array[String]) {
-    val foundKinesis = Try(Class.forName("org.apache.spark.streaming.kinesis.KinesisUtils")).isSuccess
-    if (!foundKinesis) {
-      println("Kinesis not loaded via kinesis-asl")
-      System.exit(-1)
-    }
-  }
-}
-// scalastyle:on println
diff --git a/dev/audit-release/sbt_app_sql/build.sbt b/dev/audit-release/sbt_app_sql/build.sbt
deleted file mode 100644
index 9116180f71..0000000000
--- a/dev/audit-release/sbt_app_sql/build.sbt
+++ /dev/null
@@ -1,28 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements.  See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License.  You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-name := "Simple Project"
-
-version := "1.0"
-
-scalaVersion := System.getenv.get("SCALA_VERSION")
-
-libraryDependencies += "org.apache.spark" %% "spark-sql" % System.getenv.get("SPARK_VERSION")
-
-resolvers ++= Seq(
-  "Spark Release Repository" at System.getenv.get("SPARK_RELEASE_REPOSITORY"),
-  "Spray Repository" at "http://repo.spray.cc/")
diff --git a/dev/audit-release/sbt_app_sql/src/main/scala/SqlApp.scala b/dev/audit-release/sbt_app_sql/src/main/scala/SqlApp.scala
deleted file mode 100644
index 10026314ef..0000000000
--- a/dev/audit-release/sbt_app_sql/src/main/scala/SqlApp.scala
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package main.scala
-
-import scala.collection.mutable.{ListBuffer, Queue}
-
-import org.apache.spark.SparkConf
-import org.apache.spark.SparkContext
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.SQLContext
-
-case class Person(name: String, age: Int)
-
-object SparkSqlExample {
-
-  def main(args: Array[String]) {
-    val conf = sys.env.get("SPARK_AUDIT_MASTER") match {
-      case Some(master) => new SparkConf().setAppName("Simple Sql App").setMaster(master)
-      case None => new SparkConf().setAppName("Simple Sql App")
-    }
-    val sc = new SparkContext(conf)
-    val sqlContext = new SQLContext(sc)
-
-    import sqlContext.implicits._
-    import sqlContext._
-
-    val people = sc.makeRDD(1 to 100, 10).map(x => Person(s"Name$x", x)).toDF()
-    people.createOrReplaceTempView("people")
-    val teenagers = sql("SELECT name FROM people WHERE age >= 13 AND age <= 19")
-    val teenagerNames = teenagers.map(t => "Name: " + t(0)).collect()
-    teenagerNames.foreach(println)
-
-    def test(f: => Boolean, failureMsg: String) = {
-      if (!f) {
-        println(failureMsg)
-        System.exit(-1)
-      }
-    }
-
-    test(teenagerNames.size == 7, "Unexpected number of selected elements: " + teenagerNames)
-    println("Test succeeded")
-    sc.stop()
-  }
-}
-// scalastyle:on println
diff --git a/dev/audit-release/sbt_app_streaming/build.sbt b/dev/audit-release/sbt_app_streaming/build.sbt
deleted file mode 100644
index cb369d516d..0000000000
--- a/dev/audit-release/sbt_app_streaming/build.sbt
+++ /dev/null
@@ -1,28 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements.  See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License.  You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-name := "Simple Project"
-
-version := "1.0"
-
-scalaVersion := System.getenv.get("SCALA_VERSION")
-
-libraryDependencies += "org.apache.spark" %% "spark-streaming" % System.getenv.get("SPARK_VERSION")
-
-resolvers ++= Seq(
-  "Spark Release Repository" at System.getenv.get("SPARK_RELEASE_REPOSITORY"),
-  "Spray Repository" at "http://repo.spray.cc/")
diff --git a/dev/audit-release/sbt_app_streaming/src/main/scala/StreamingApp.scala b/dev/audit-release/sbt_app_streaming/src/main/scala/StreamingApp.scala
deleted file mode 100644
index d6a074687f..0000000000
--- a/dev/audit-release/sbt_app_streaming/src/main/scala/StreamingApp.scala
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package main.scala
-
-import scala.collection.mutable.{ListBuffer, Queue}
-
-import org.apache.spark.SparkConf
-import org.apache.spark.rdd.RDD
-import org.apache.spark.streaming.StreamingContext
-import org.apache.spark.streaming._
-
-object SparkStreamingExample {
-
-  def main(args: Array[String]) {
-    val conf = sys.env.get("SPARK_AUDIT_MASTER") match {
-      case Some(master) => new SparkConf().setAppName("Simple Streaming App").setMaster(master)
-      case None => new SparkConf().setAppName("Simple Streaming App")
-    }
-    val ssc = new StreamingContext(conf, Seconds(1))
-    val seen = ListBuffer[RDD[Int]]()
-
-    val rdd1 = ssc.sparkContext.makeRDD(1 to 100, 10)
-    val rdd2 = ssc.sparkContext.makeRDD(1 to 1000, 10)
-    val rdd3 = ssc.sparkContext.makeRDD(1 to 10000, 10)
-
-    val queue = Queue(rdd1, rdd2, rdd3)
-    val stream = ssc.queueStream(queue)
-
-    stream.foreachRDD(rdd => seen += rdd)
-    ssc.start()
-    Thread.sleep(5000)
-
-    def test(f: => Boolean, failureMsg: String) = {
-      if (!f) {
-        println(failureMsg)
-        System.exit(-1)
-      }
-    }
-
-    val rddCounts = seen.map(rdd => rdd.count()).filter(_ > 0)
-    test(rddCounts.length == 3, "Did not collect three RDD's from stream")
-    test(rddCounts.toSet == Set(100, 1000, 10000), "Did not find expected streams")
-
-    println("Test succeeded")
-
-    ssc.stop()
-  }
-}
-// scalastyle:on println