diff options
author | Thomas Dudziak <tomdzk@gmail.com> | 2012-12-18 10:44:03 -0800 |
---|---|---|
committer | Thomas Dudziak <tomdzk@gmail.com> | 2012-12-18 12:08:19 -0800 |
commit | 4af6cad37a256fe958e8da9e0937d359bdd5dec5 (patch) | |
tree | 62ef0c035b48efa8b8df81af3a0322fc92e62f43 /repl | |
parent | b82a6dd2c72d6555aeaa2b523ddf564434f5e10c (diff) | |
download | spark-4af6cad37a256fe958e8da9e0937d359bdd5dec5.tar.gz spark-4af6cad37a256fe958e8da9e0937d359bdd5dec5.tar.bz2 spark-4af6cad37a256fe958e8da9e0937d359bdd5dec5.zip |
Fixed repl maven build to produce artifacts with the appropriate hadoop classifier and extracted repl fat-jar and debian packaging into a separate project to make Maven happy
Diffstat (limited to 'repl')
-rw-r--r-- | repl/pom.xml | 146 | ||||
-rwxr-xr-x | repl/src/deb/bin/run | 41 | ||||
-rwxr-xr-x | repl/src/deb/bin/spark-executor | 5 | ||||
-rwxr-xr-x | repl/src/deb/bin/spark-shell | 4 | ||||
-rw-r--r-- | repl/src/deb/control/control | 8 |
5 files changed, 7 insertions, 197 deletions
diff --git a/repl/pom.xml b/repl/pom.xml index f6df4ba9f7..114e3e9932 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -99,46 +99,17 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-core</artifactId> + <scope>provided</scope> </dependency> </dependencies> <build> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> + <artifactId>maven-jar-plugin</artifactId> <configuration> - <shadedArtifactAttached>true</shadedArtifactAttached> - <shadedClassifierName>shaded-hadoop1</shadedClassifierName> - <filters> - <filter> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> + <classifier>hadoop1</classifier> </configuration> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> - <configuration> - <transformers> - <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> - <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer"> - <resource>reference.conf</resource> - </transformer> - <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> - <mainClass>spark.repl.Main</mainClass> - </transformer> - </transformers> - </configuration> - </execution> - </executions> </plugin> </plugins> </build> @@ -172,125 +143,22 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-core</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> + <scope>provided</scope> </dependency> </dependencies> <build> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> + <artifactId>maven-jar-plugin</artifactId> <configuration> - <shadedArtifactAttached>true</shadedArtifactAttached> - <shadedClassifierName>shaded-hadoop2</shadedClassifierName> - <filters> - <filter> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> + <classifier>hadoop2</classifier> </configuration> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> - <configuration> - <transformers> - <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> - <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer"> - <resource>reference.conf</resource> - </transformer> - <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> - <mainClass>spark.repl.Main</mainClass> - </transformer> - </transformers> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - <profile> - <id>deb</id> - <build> - <plugins> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>buildnumber-maven-plugin</artifactId> - <version>1.1</version> - <executions> - <execution> - <phase>validate</phase> - <goals> - <goal>create</goal> - </goals> - <configuration> - <shortRevisionLength>8</shortRevisionLength> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.vafer</groupId> - <artifactId>jdeb</artifactId> - <version>0.11</version> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>jdeb</goal> - </goals> - <configuration> - <deb>${project.build.directory}/${project.artifactId}-${classifier}_${project.version}-${buildNumber}_all.deb</deb> - <attach>false</attach> - <compression>gzip</compression> - <dataSet> - <data> - <src>${project.build.directory}/${project.artifactId}-${project.version}-shaded-${classifier}.jar</src> - <type>file</type> - <mapper> - <type>perm</type> - <user>${deb.user}</user> - <group>${deb.user}</group> - <prefix>${deb.install.path}</prefix> - </mapper> - </data> - <data> - <src>${basedir}/src/deb/bin</src> - <type>directory</type> - <mapper> - <type>perm</type> - <user>${deb.user}</user> - <group>${deb.user}</group> - <prefix>${deb.install.path}</prefix> - <filemode>744</filemode> - </mapper> - </data> - <data> - <src>${basedir}/../conf</src> - <type>directory</type> - <mapper> - <type>perm</type> - <user>${deb.user}</user> - <group>${deb.user}</group> - <prefix>${deb.install.path}/conf</prefix> - <filemode>744</filemode> - </mapper> - </data> - </dataSet> - </configuration> - </execution> - </executions> </plugin> </plugins> </build> diff --git a/repl/src/deb/bin/run b/repl/src/deb/bin/run deleted file mode 100755 index c54c9e97a0..0000000000 --- a/repl/src/deb/bin/run +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -SCALA_VERSION=2.9.2 - -# Figure out where the Scala framework is installed -FWDIR="$(cd `dirname $0`; pwd)" - -# Export this as SPARK_HOME -export SPARK_HOME="$FWDIR" - -# Load environment variables from conf/spark-env.sh, if it exists -if [ -e $FWDIR/conf/spark-env.sh ] ; then - . $FWDIR/conf/spark-env.sh -fi - -# Figure out how much memory to use per executor and set it as an environment -# variable so that our process sees it and can report it to Mesos -if [ -z "$SPARK_MEM" ] ; then - SPARK_MEM="512m" -fi -export SPARK_MEM - -# Set JAVA_OPTS to be able to load native libraries and to set heap size -JAVA_OPTS="$SPARK_JAVA_OPTS" -JAVA_OPTS+=" -Djava.library.path=$SPARK_LIBRARY_PATH" -JAVA_OPTS+=" -Xms$SPARK_MEM -Xmx$SPARK_MEM" -# Load extra JAVA_OPTS from conf/java-opts, if it exists -if [ -e $FWDIR/conf/java-opts ] ; then - JAVA_OPTS+=" `cat $FWDIR/conf/java-opts`" -fi -export JAVA_OPTS - -# Build up classpath -CLASSPATH="$SPARK_CLASSPATH" -CLASSPATH+=":$FWDIR/conf" -for jar in `find $FWDIR -name '*jar'`; do - CLASSPATH+=":$jar" -done -export CLASSPATH - -exec java -Dscala.usejavacp=true -Djline.shutdownhook=true -cp "$CLASSPATH" $JAVA_OPTS $EXTRA_ARGS "$@" diff --git a/repl/src/deb/bin/spark-executor b/repl/src/deb/bin/spark-executor deleted file mode 100755 index 47b9cccdfe..0000000000 --- a/repl/src/deb/bin/spark-executor +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -FWDIR="$(cd `dirname $0`; pwd)" -echo "Running spark-executor with framework dir = $FWDIR" -exec $FWDIR/run spark.executor.MesosExecutorBackend diff --git a/repl/src/deb/bin/spark-shell b/repl/src/deb/bin/spark-shell deleted file mode 100755 index 219c66eb0b..0000000000 --- a/repl/src/deb/bin/spark-shell +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -FWDIR="$(cd `dirname $0`; pwd)" -exec $FWDIR/run spark.repl.Main "$@" diff --git a/repl/src/deb/control/control b/repl/src/deb/control/control deleted file mode 100644 index 6586986c76..0000000000 --- a/repl/src/deb/control/control +++ /dev/null @@ -1,8 +0,0 @@ -Package: spark-repl -Version: [[version]]-[[buildNumber]] -Section: misc -Priority: extra -Architecture: all -Maintainer: Matei Zaharia <matei.zaharia@gmail.com> -Description: spark repl -Distribution: development |