From 4af6cad37a256fe958e8da9e0937d359bdd5dec5 Mon Sep 17 00:00:00 2001 From: Thomas Dudziak Date: Tue, 18 Dec 2012 10:44:03 -0800 Subject: Fixed repl maven build to produce artifacts with the appropriate hadoop classifier and extracted repl fat-jar and debian packaging into a separate project to make Maven happy --- repl-bin/pom.xml | 231 ++++++++++++++++++++++++++++++++++++ repl-bin/src/deb/bin/run | 41 +++++++ repl-bin/src/deb/bin/spark-executor | 5 + repl-bin/src/deb/bin/spark-shell | 4 + repl-bin/src/deb/control/control | 8 ++ 5 files changed, 289 insertions(+) create mode 100644 repl-bin/pom.xml create mode 100755 repl-bin/src/deb/bin/run create mode 100755 repl-bin/src/deb/bin/spark-executor create mode 100755 repl-bin/src/deb/bin/spark-shell create mode 100644 repl-bin/src/deb/control/control (limited to 'repl-bin') diff --git a/repl-bin/pom.xml b/repl-bin/pom.xml new file mode 100644 index 0000000000..72a946f3d7 --- /dev/null +++ b/repl-bin/pom.xml @@ -0,0 +1,231 @@ + + + 4.0.0 + + org.spark-project + parent + 0.7.0-SNAPSHOT + ../pom.xml + + + org.spark-project + spark-repl-bin + pom + Spark Project REPL binary packaging + http://spark-project.org/ + + + /usr/share/spark + root + + + + + + org.apache.maven.plugins + maven-shade-plugin + + false + ${project.build.directory}/${project.artifactId}-${project.version}-shaded-${classifier}.jar + + + *:* + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + package + + shade + + + + + + reference.conf + + + spark.repl.Main + + + + + + + + + + + + hadoop1 + + hadoop1 + + + + org.spark-project + spark-core + ${project.version} + hadoop1 + + + org.spark-project + spark-bagel + ${project.version} + hadoop1 + runtime + + + org.spark-project + spark-examples + ${project.version} + hadoop1 + runtime + + + org.spark-project + spark-repl + ${project.version} + hadoop1 + runtime + + + org.apache.hadoop + hadoop-core + runtime + + + + + hadoop2 + + hadoop2 + + + + org.spark-project + spark-core + ${project.version} + hadoop2 + + + org.spark-project + spark-bagel + ${project.version} + hadoop2 + runtime + + + org.spark-project + spark-examples + ${project.version} + hadoop2 + runtime + + + org.spark-project + spark-repl + ${project.version} + hadoop2 + runtime + + + org.apache.hadoop + hadoop-core + runtime + + + org.apache.hadoop + hadoop-client + runtime + + + + + deb + + + + org.codehaus.mojo + buildnumber-maven-plugin + 1.1 + + + validate + + create + + + 8 + + + + + + org.vafer + jdeb + 0.11 + + + package + + jdeb + + + ${project.build.directory}/${project.artifactId}-${classifier}_${project.version}-${buildNumber}_all.deb + false + gzip + + + ${project.build.directory}/${project.artifactId}-${project.version}-shaded-${classifier}.jar + file + + perm + ${deb.user} + ${deb.user} + ${deb.install.path} + + + + ${basedir}/src/deb/bin + directory + + perm + ${deb.user} + ${deb.user} + ${deb.install.path} + 744 + + + + ${basedir}/../conf + directory + + perm + ${deb.user} + ${deb.user} + ${deb.install.path}/conf + 744 + + + + + + + + + + + + diff --git a/repl-bin/src/deb/bin/run b/repl-bin/src/deb/bin/run new file mode 100755 index 0000000000..c54c9e97a0 --- /dev/null +++ b/repl-bin/src/deb/bin/run @@ -0,0 +1,41 @@ +#!/bin/bash + +SCALA_VERSION=2.9.2 + +# Figure out where the Scala framework is installed +FWDIR="$(cd `dirname $0`; pwd)" + +# Export this as SPARK_HOME +export SPARK_HOME="$FWDIR" + +# Load environment variables from conf/spark-env.sh, if it exists +if [ -e $FWDIR/conf/spark-env.sh ] ; then + . $FWDIR/conf/spark-env.sh +fi + +# Figure out how much memory to use per executor and set it as an environment +# variable so that our process sees it and can report it to Mesos +if [ -z "$SPARK_MEM" ] ; then + SPARK_MEM="512m" +fi +export SPARK_MEM + +# Set JAVA_OPTS to be able to load native libraries and to set heap size +JAVA_OPTS="$SPARK_JAVA_OPTS" +JAVA_OPTS+=" -Djava.library.path=$SPARK_LIBRARY_PATH" +JAVA_OPTS+=" -Xms$SPARK_MEM -Xmx$SPARK_MEM" +# Load extra JAVA_OPTS from conf/java-opts, if it exists +if [ -e $FWDIR/conf/java-opts ] ; then + JAVA_OPTS+=" `cat $FWDIR/conf/java-opts`" +fi +export JAVA_OPTS + +# Build up classpath +CLASSPATH="$SPARK_CLASSPATH" +CLASSPATH+=":$FWDIR/conf" +for jar in `find $FWDIR -name '*jar'`; do + CLASSPATH+=":$jar" +done +export CLASSPATH + +exec java -Dscala.usejavacp=true -Djline.shutdownhook=true -cp "$CLASSPATH" $JAVA_OPTS $EXTRA_ARGS "$@" diff --git a/repl-bin/src/deb/bin/spark-executor b/repl-bin/src/deb/bin/spark-executor new file mode 100755 index 0000000000..47b9cccdfe --- /dev/null +++ b/repl-bin/src/deb/bin/spark-executor @@ -0,0 +1,5 @@ +#!/bin/bash + +FWDIR="$(cd `dirname $0`; pwd)" +echo "Running spark-executor with framework dir = $FWDIR" +exec $FWDIR/run spark.executor.MesosExecutorBackend diff --git a/repl-bin/src/deb/bin/spark-shell b/repl-bin/src/deb/bin/spark-shell new file mode 100755 index 0000000000..219c66eb0b --- /dev/null +++ b/repl-bin/src/deb/bin/spark-shell @@ -0,0 +1,4 @@ +#!/bin/bash + +FWDIR="$(cd `dirname $0`; pwd)" +exec $FWDIR/run spark.repl.Main "$@" diff --git a/repl-bin/src/deb/control/control b/repl-bin/src/deb/control/control new file mode 100644 index 0000000000..afadb3fbfe --- /dev/null +++ b/repl-bin/src/deb/control/control @@ -0,0 +1,8 @@ +Package: [[artifactId]] +Version: [[version]]-[[buildNumber]] +Section: misc +Priority: extra +Architecture: all +Maintainer: Matei Zaharia +Description: spark repl +Distribution: development -- cgit v1.2.3