diff options
author | Patrick Wendell <pwendell@gmail.com> | 2015-01-09 09:40:18 -0800 |
---|---|---|
committer | Patrick Wendell <pwendell@gmail.com> | 2015-01-09 09:40:18 -0800 |
commit | 1790b38695b46400a24b0b7e278e8e8388748211 (patch) | |
tree | ef128dd5a23235c7e7e1193244fe1487300f26cb | |
parent | 547df97715580f99ae573a49a86da12bf20cbc3d (diff) | |
download | spark-1790b38695b46400a24b0b7e278e8e8388748211.tar.gz spark-1790b38695b46400a24b0b7e278e8e8388748211.tar.bz2 spark-1790b38695b46400a24b0b7e278e8e8388748211.zip |
HOTFIX: Minor improvements to make-distribution.sh
1. Renames $FWDIR to $SPARK_HOME (vast majority of diff).
2. Use Spark-provided Maven.
3. Logs build flags in the RELEASE file.
Author: Patrick Wendell <pwendell@gmail.com>
Closes #3973 from pwendell/master and squashes the following commits:
340a2fa [Patrick Wendell] HOTFIX: Minor improvements to make-distribution.sh
-rwxr-xr-x | make-distribution.sh | 61 |
1 files changed, 34 insertions, 27 deletions
diff --git a/make-distribution.sh b/make-distribution.sh index 45c99e42e5..4e2f400be3 100755 --- a/make-distribution.sh +++ b/make-distribution.sh @@ -28,18 +28,20 @@ set -o pipefail set -e # Figure out where the Spark framework is installed -FWDIR="$(cd "`dirname "$0"`"; pwd)" -DISTDIR="$FWDIR/dist" +SPARK_HOME="$(cd "`dirname "$0"`"; pwd)" +DISTDIR="$SPARK_HOME/dist" SPARK_TACHYON=false MAKE_TGZ=false NAME=none +MVN="$SPARK_HOME/build/mvn" function exit_with_usage { echo "make-distribution.sh - tool for making binary distributions of Spark" echo "" echo "usage:" - echo "./make-distribution.sh [--name] [--tgz] [--with-tachyon] <maven build options>" + cl_options="[--name] [--tgz] [--mvn <mvn-command>] [--with-tachyon]" + echo "./make-distribution.sh $cl_options <maven build options>" echo "See Spark's \"Building Spark\" doc for correct Maven options." echo "" exit 1 @@ -71,6 +73,10 @@ while (( "$#" )); do --tgz) MAKE_TGZ=true ;; + --mvn) + MVN="$2" + shift + ;; --name) NAME="$2" shift @@ -109,9 +115,9 @@ if which git &>/dev/null; then unset GITREV fi -if ! which mvn &>/dev/null; then - echo -e "You need Maven installed to build Spark." - echo -e "Download Maven from https://maven.apache.org/" +if ! which $MVN &>/dev/null; then + echo -e "Could not locate Maven command: '$MVN'." + echo -e "Specify the Maven command with the --mvn flag" exit -1; fi @@ -119,7 +125,7 @@ VERSION=$(mvn help:evaluate -Dexpression=project.version 2>/dev/null | grep -v " SPARK_HADOOP_VERSION=$(mvn help:evaluate -Dexpression=hadoop.version $@ 2>/dev/null\ | grep -v "INFO"\ | tail -n 1) -SPARK_HIVE=$(mvn help:evaluate -Dexpression=project.activeProfiles -pl sql/hive $@ 2>/dev/null\ +SPARK_HIVE=$($MVN help:evaluate -Dexpression=project.activeProfiles -pl sql/hive $@ 2>/dev/null\ | grep -v "INFO"\ | fgrep --count "<id>hive</id>";\ # Reset exit status to 0, otherwise the script stops here if the last grep finds nothing\ @@ -161,11 +167,11 @@ else fi # Build uber fat JAR -cd "$FWDIR" +cd "$SPARK_HOME" export MAVEN_OPTS="-Xmx2g -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=512m" -BUILD_COMMAND="mvn clean package -DskipTests $@" +BUILD_COMMAND="$MVN clean package -DskipTests $@" # Actually build the jar echo -e "\nBuilding with..." @@ -177,41 +183,42 @@ ${BUILD_COMMAND} rm -rf "$DISTDIR" mkdir -p "$DISTDIR/lib" echo "Spark $VERSION$GITREVSTRING built for Hadoop $SPARK_HADOOP_VERSION" > "$DISTDIR/RELEASE" +echo "Build flags: $@" >> "$DISTDIR/RELEASE" # Copy jars -cp "$FWDIR"/assembly/target/scala*/*assembly*hadoop*.jar "$DISTDIR/lib/" -cp "$FWDIR"/examples/target/scala*/spark-examples*.jar "$DISTDIR/lib/" +cp "$SPARK_HOME"/assembly/target/scala*/*assembly*hadoop*.jar "$DISTDIR/lib/" +cp "$SPARK_HOME"/examples/target/scala*/spark-examples*.jar "$DISTDIR/lib/" # This will fail if the -Pyarn profile is not provided # In this case, silence the error and ignore the return code of this command -cp "$FWDIR"/network/yarn/target/scala*/spark-*-yarn-shuffle.jar "$DISTDIR/lib/" &> /dev/null || : +cp "$SPARK_HOME"/network/yarn/target/scala*/spark-*-yarn-shuffle.jar "$DISTDIR/lib/" &> /dev/null || : # Copy example sources (needed for python and SQL) mkdir -p "$DISTDIR/examples/src/main" -cp -r "$FWDIR"/examples/src/main "$DISTDIR/examples/src/" +cp -r "$SPARK_HOME"/examples/src/main "$DISTDIR/examples/src/" if [ "$SPARK_HIVE" == "1" ]; then - cp "$FWDIR"/lib_managed/jars/datanucleus*.jar "$DISTDIR/lib/" + cp "$SPARK_HOME"/lib_managed/jars/datanucleus*.jar "$DISTDIR/lib/" fi # Copy license and ASF files -cp "$FWDIR/LICENSE" "$DISTDIR" -cp "$FWDIR/NOTICE" "$DISTDIR" +cp "$SPARK_HOME/LICENSE" "$DISTDIR" +cp "$SPARK_HOME/NOTICE" "$DISTDIR" -if [ -e "$FWDIR"/CHANGES.txt ]; then - cp "$FWDIR/CHANGES.txt" "$DISTDIR" +if [ -e "$SPARK_HOME"/CHANGES.txt ]; then + cp "$SPARK_HOME/CHANGES.txt" "$DISTDIR" fi # Copy data files -cp -r "$FWDIR/data" "$DISTDIR" +cp -r "$SPARK_HOME/data" "$DISTDIR" # Copy other things mkdir "$DISTDIR"/conf -cp "$FWDIR"/conf/*.template "$DISTDIR"/conf -cp "$FWDIR/README.md" "$DISTDIR" -cp -r "$FWDIR/bin" "$DISTDIR" -cp -r "$FWDIR/python" "$DISTDIR" -cp -r "$FWDIR/sbin" "$DISTDIR" -cp -r "$FWDIR/ec2" "$DISTDIR" +cp "$SPARK_HOME"/conf/*.template "$DISTDIR"/conf +cp "$SPARK_HOME/README.md" "$DISTDIR" +cp -r "$SPARK_HOME/bin" "$DISTDIR" +cp -r "$SPARK_HOME/python" "$DISTDIR" +cp -r "$SPARK_HOME/sbin" "$DISTDIR" +cp -r "$SPARK_HOME/ec2" "$DISTDIR" # Download and copy in tachyon, if requested if [ "$SPARK_TACHYON" == "true" ]; then @@ -243,9 +250,9 @@ fi if [ "$MAKE_TGZ" == "true" ]; then TARDIR_NAME=spark-$VERSION-bin-$NAME - TARDIR="$FWDIR/$TARDIR_NAME" + TARDIR="$SPARK_HOME/$TARDIR_NAME" rm -rf "$TARDIR" cp -r "$DISTDIR" "$TARDIR" - tar czf "spark-$VERSION-bin-$NAME.tgz" -C "$FWDIR" "$TARDIR_NAME" + tar czf "spark-$VERSION-bin-$NAME.tgz" -C "$SPARK_HOME" "$TARDIR_NAME" rm -rf "$TARDIR" fi |