diff options
| -rw-r--r-- | .gitignore | 3 | ||||
| -rw-r--r-- | dev/.rat-excludes (renamed from .rat-excludes) | 0 | ||||
| -rwxr-xr-x | dev/check-license | 7 | ||||
| -rw-r--r-- | dev/checkstyle-suppressions.xml (renamed from checkstyle-suppressions.xml) | 0 | ||||
| -rw-r--r-- | dev/checkstyle.xml (renamed from checkstyle.xml) | 0 | ||||
| -rwxr-xr-x | dev/create-release/release-build.sh | 2 | ||||
| -rwxr-xr-x | dev/lint-python | 6 | ||||
| -rwxr-xr-x | dev/make-distribution.sh (renamed from make-distribution.sh) | 6 | ||||
| -rw-r--r-- | dev/tox.ini (renamed from tox.ini) | 0 | ||||
| -rw-r--r-- | docs/building-spark.md | 6 | ||||
| -rw-r--r-- | docs/running-on-mesos.md | 4 | ||||
| -rw-r--r-- | pom.xml | 4 | ||||
| -rw-r--r-- | python/pylintrc (renamed from pylintrc) | 0 | ||||
| -rwxr-xr-x | sbt/sbt | 29 |
14 files changed, 18 insertions, 49 deletions
diff --git a/.gitignore b/.gitignore index 8ecf536e79..05afbb5e5e 100644 --- a/.gitignore +++ b/.gitignore @@ -17,8 +17,6 @@ cache work/ out/ .DS_Store -third_party/libmesos.so -third_party/libmesos.dylib build/apache-maven* build/zinc* build/scala* @@ -60,7 +58,6 @@ dev/create-release/*final spark-*-bin-*.tgz unit-tests.log /lib/ -rat-results.txt scalastyle.txt scalastyle-output.xml R-unit-tests.log diff --git a/.rat-excludes b/dev/.rat-excludes index 8b5061415f..8b5061415f 100644 --- a/.rat-excludes +++ b/dev/.rat-excludes diff --git a/dev/check-license b/dev/check-license index 10740cfdc5..678e73fd60 100755 --- a/dev/check-license +++ b/dev/check-license @@ -58,7 +58,7 @@ else declare java_cmd=java fi -export RAT_VERSION=0.10 +export RAT_VERSION=0.11 export rat_jar="$FWDIR"/lib/apache-rat-${RAT_VERSION}.jar mkdir -p "$FWDIR"/lib @@ -67,14 +67,15 @@ mkdir -p "$FWDIR"/lib exit 1 } -$java_cmd -jar "$rat_jar" -E "$FWDIR"/.rat-excludes -d "$FWDIR" > rat-results.txt +mkdir target +$java_cmd -jar "$rat_jar" -E "$FWDIR"/dev/.rat-excludes -d "$FWDIR" > target/rat-results.txt if [ $? -ne 0 ]; then echo "RAT exited abnormally" exit 1 fi -ERRORS="$(cat rat-results.txt | grep -e "??")" +ERRORS="$(cat target/rat-results.txt | grep -e "??")" if test ! -z "$ERRORS"; then echo "Could not find Apache license headers in the following files:" diff --git a/checkstyle-suppressions.xml b/dev/checkstyle-suppressions.xml index 9242be3d03..9242be3d03 100644 --- a/checkstyle-suppressions.xml +++ b/dev/checkstyle-suppressions.xml diff --git a/checkstyle.xml b/dev/checkstyle.xml index a165fee2a4..a165fee2a4 100644 --- a/checkstyle.xml +++ b/dev/checkstyle.xml diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh index c08b6d7de6..65e80fc760 100755 --- a/dev/create-release/release-build.sh +++ b/dev/create-release/release-build.sh @@ -165,7 +165,7 @@ if [[ "$1" == "package" ]]; then # Get maven home set by MVN MVN_HOME=`$MVN -version 2>&1 | grep 'Maven home' | awk '{print $NF}'` - ./make-distribution.sh --name $NAME --mvn $MVN_HOME/bin/mvn --tgz $FLAGS \ + ./dev/make-distribution.sh --name $NAME --mvn $MVN_HOME/bin/mvn --tgz $FLAGS \ -DzincPort=$ZINC_PORT 2>&1 > ../binary-release-$NAME.log cd .. cp spark-$SPARK_VERSION-bin-$NAME/spark-$SPARK_VERSION-bin-$NAME.tgz . diff --git a/dev/lint-python b/dev/lint-python index 068337d273..477ac0ef6d 100755 --- a/dev/lint-python +++ b/dev/lint-python @@ -37,7 +37,7 @@ compile_status="${PIPESTATUS[0]}" #+ See: https://github.com/apache/spark/pull/1744#issuecomment-50982162 #+ TODOs: #+ - Download pep8 from PyPI. It's more "official". -PEP8_VERSION="1.6.2" +PEP8_VERSION="1.7.0" PEP8_SCRIPT_PATH="$SPARK_ROOT_DIR/dev/pep8-$PEP8_VERSION.py" PEP8_SCRIPT_REMOTE_PATH="https://raw.githubusercontent.com/jcrocholl/pep8/$PEP8_VERSION/pep8.py" @@ -80,7 +80,7 @@ export "PATH=$PYTHONPATH:$PATH" #+ first, but we do so so that the check status can #+ be output before the report, like with the #+ scalastyle and RAT checks. -python "$PEP8_SCRIPT_PATH" --ignore=E402,E731,E241,W503,E226 $PATHS_TO_CHECK >> "$PEP8_REPORT_PATH" +python "$PEP8_SCRIPT_PATH" --ignore=E402,E731,E241,W503,E226 --config=dev/tox.ini $PATHS_TO_CHECK >> "$PEP8_REPORT_PATH" pep8_status="${PIPESTATUS[0]}" if [ "$compile_status" -eq 0 -a "$pep8_status" -eq 0 ]; then @@ -122,7 +122,7 @@ fi # for to_be_checked in "$PATHS_TO_CHECK" # do -# pylint --rcfile="$SPARK_ROOT_DIR/pylintrc" $to_be_checked >> "$PYLINT_REPORT_PATH" +# pylint --rcfile="$SPARK_ROOT_DIR/python/pylintrc" $to_be_checked >> "$PYLINT_REPORT_PATH" # done # if [ "${PIPESTATUS[0]}" -ne 0 ]; then diff --git a/make-distribution.sh b/dev/make-distribution.sh index ac90ea317a..ac4e9b90f0 100755 --- a/make-distribution.sh +++ b/dev/make-distribution.sh @@ -29,7 +29,7 @@ set -e set -x # Figure out where the Spark framework is installed -SPARK_HOME="$(cd "`dirname "$0"`"; pwd)" +SPARK_HOME="$(cd "`dirname "$0"`/.."; pwd)" DISTDIR="$SPARK_HOME/dist" MAKE_TGZ=false @@ -41,7 +41,7 @@ function exit_with_usage { echo "" echo "usage:" cl_options="[--name] [--tgz] [--mvn <mvn-command>]" - echo "./make-distribution.sh $cl_options <maven build options>" + echo "make-distribution.sh $cl_options <maven build options>" echo "See Spark's \"Building Spark\" doc for correct Maven options." echo "" exit 1 @@ -104,7 +104,7 @@ fi if [ $(command -v git) ]; then GITREV=$(git rev-parse --short HEAD 2>/dev/null || :) if [ ! -z "$GITREV" ]; then - GITREVSTRING=" (git revision $GITREV)" + GITREVSTRING=" (git revision $GITREV)" fi unset GITREV fi diff --git a/docs/building-spark.md b/docs/building-spark.md index adf798847c..2c6294133e 100644 --- a/docs/building-spark.md +++ b/docs/building-spark.md @@ -35,12 +35,12 @@ to the `sharedSettings` val. See also [this PR](https://github.com/apache/spark/ To create a Spark distribution like those distributed by the [Spark Downloads](http://spark.apache.org/downloads.html) page, and that is laid out so as -to be runnable, use `make-distribution.sh` in the project root directory. It can be configured +to be runnable, use `./dev/make-distribution.sh` in the project root directory. It can be configured with Maven profile settings and so on like the direct Maven build. Example: - ./make-distribution.sh --name custom-spark --tgz -Psparkr -Phadoop-2.4 -Phive -Phive-thriftserver -Pyarn + ./dev/make-distribution.sh --name custom-spark --tgz -Psparkr -Phadoop-2.4 -Phive -Phive-thriftserver -Pyarn -For more information on usage, run `./make-distribution.sh --help` +For more information on usage, run `./dev/make-distribution.sh --help` # Setting up Maven's Memory Usage diff --git a/docs/running-on-mesos.md b/docs/running-on-mesos.md index 9816d030e9..912a010812 100644 --- a/docs/running-on-mesos.md +++ b/docs/running-on-mesos.md @@ -98,10 +98,10 @@ To host on HDFS, use the Hadoop fs put command: `hadoop fs -put spark-{{site.SPA Or if you are using a custom-compiled version of Spark, you will need to create a package using -the `make-distribution.sh` script included in a Spark source tarball/checkout. +the `dev/make-distribution.sh` script included in a Spark source tarball/checkout. 1. Download and build Spark using the instructions [here](index.html) -2. Create a binary package using `make-distribution.sh --tgz`. +2. Create a binary package using `./dev/make-distribution.sh --tgz`. 3. Upload archive to http/s3/hdfs @@ -2246,7 +2246,7 @@ <failOnWarning>false</failOnWarning> <sourceDirectory>${basedir}/src/main/scala</sourceDirectory> <testSourceDirectory>${basedir}/src/test/scala</testSourceDirectory> - <configLocation>scalastyle-config.xml</configLocation> + <configLocation>dev/scalastyle-config.xml</configLocation> <outputFile>${basedir}/target/scalastyle-output.xml</outputFile> <inputEncoding>${project.build.sourceEncoding}</inputEncoding> <outputEncoding>${project.reporting.outputEncoding}</outputEncoding> @@ -2270,7 +2270,7 @@ <failOnWarning>false</failOnWarning> <sourceDirectory>${basedir}/src/main/java</sourceDirectory> <testSourceDirectory>${basedir}/src/test/java</testSourceDirectory> - <configLocation>checkstyle.xml</configLocation> + <configLocation>dev/checkstyle.xml</configLocation> <outputFile>${basedir}/target/checkstyle-output.xml</outputFile> <inputEncoding>${project.build.sourceEncoding}</inputEncoding> <outputEncoding>${project.reporting.outputEncoding}</outputEncoding> diff --git a/pylintrc b/python/pylintrc index 6a675770da..6a675770da 100644 --- a/pylintrc +++ b/python/pylintrc diff --git a/sbt/sbt b/sbt/sbt deleted file mode 100755 index 41438251f6..0000000000 --- a/sbt/sbt +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Determine the current working directory -_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -echo "NOTE: The sbt/sbt script has been relocated to build/sbt." >&2 -echo " Please update references to point to the new location." >&2 -echo "" >&2 -echo " Invoking 'build/sbt $@' now ..." >&2 -echo "" >&2 - -${_DIR}/../build/sbt "$@" |