diff options
| author | Thomas Dudziak <tomdzk@gmail.com> | 2012-12-18 10:44:03 -0800 |
|---|---|---|
| committer | Thomas Dudziak <tomdzk@gmail.com> | 2012-12-18 12:08:19 -0800 |
| commit | 4af6cad37a256fe958e8da9e0937d359bdd5dec5 (patch) | |
| tree | 62ef0c035b48efa8b8df81af3a0322fc92e62f43 | |
| parent | b82a6dd2c72d6555aeaa2b523ddf564434f5e10c (diff) | |
| download | spark-4af6cad37a256fe958e8da9e0937d359bdd5dec5.tar.gz spark-4af6cad37a256fe958e8da9e0937d359bdd5dec5.tar.bz2 spark-4af6cad37a256fe958e8da9e0937d359bdd5dec5.zip | |
Fixed repl maven build to produce artifacts with the appropriate hadoop classifier and extracted repl fat-jar and debian packaging into a separate project to make Maven happy
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | pom.xml | 5 | ||||
| -rw-r--r-- | repl-bin/pom.xml | 231 | ||||
| -rwxr-xr-x | repl-bin/src/deb/bin/run (renamed from repl/src/deb/bin/run) | 0 | ||||
| -rwxr-xr-x | repl-bin/src/deb/bin/spark-executor (renamed from repl/src/deb/bin/spark-executor) | 0 | ||||
| -rwxr-xr-x | repl-bin/src/deb/bin/spark-shell (renamed from repl/src/deb/bin/spark-shell) | 0 | ||||
| -rw-r--r-- | repl-bin/src/deb/control/control (renamed from repl/src/deb/control/control) | 2 | ||||
| -rw-r--r-- | repl/pom.xml | 146 |
8 files changed, 243 insertions, 142 deletions
diff --git a/.gitignore b/.gitignore index f22248f40d..c207409e3c 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ project/plugins/src_managed/ logs/ log/ spark-tests.log +dependency-reduced-pom.xml @@ -39,9 +39,10 @@ <modules> <module>core</module> - <module>repl</module> - <module>examples</module> <module>bagel</module> + <module>examples</module> + <module>repl</module> + <module>repl-bin</module> </modules> <properties> diff --git a/repl-bin/pom.xml b/repl-bin/pom.xml new file mode 100644 index 0000000000..72a946f3d7 --- /dev/null +++ b/repl-bin/pom.xml @@ -0,0 +1,231 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.spark-project</groupId> + <artifactId>parent</artifactId> + <version>0.7.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + + <groupId>org.spark-project</groupId> + <artifactId>spark-repl-bin</artifactId> + <packaging>pom</packaging> + <name>Spark Project REPL binary packaging</name> + <url>http://spark-project.org/</url> + + <properties> + <deb.install.path>/usr/share/spark</deb.install.path> + <deb.user>root</deb.user> + </properties> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <configuration> + <shadedArtifactAttached>false</shadedArtifactAttached> + <outputFile>${project.build.directory}/${project.artifactId}-${project.version}-shaded-${classifier}.jar</outputFile> + <artifactSet> + <includes> + <include>*:*</include> + </includes> + </artifactSet> + <filters> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> + </configuration> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <transformers> + <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> + <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer"> + <resource>reference.conf</resource> + </transformer> + <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> + <mainClass>spark.repl.Main</mainClass> + </transformer> + </transformers> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + + <profiles> + <profile> + <id>hadoop1</id> + <properties> + <classifier>hadoop1</classifier> + </properties> + <dependencies> + <dependency> + <groupId>org.spark-project</groupId> + <artifactId>spark-core</artifactId> + <version>${project.version}</version> + <classifier>hadoop1</classifier> + </dependency> + <dependency> + <groupId>org.spark-project</groupId> + <artifactId>spark-bagel</artifactId> + <version>${project.version}</version> + <classifier>hadoop1</classifier> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>org.spark-project</groupId> + <artifactId>spark-examples</artifactId> + <version>${project.version}</version> + <classifier>hadoop1</classifier> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>org.spark-project</groupId> + <artifactId>spark-repl</artifactId> + <version>${project.version}</version> + <classifier>hadoop1</classifier> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-core</artifactId> + <scope>runtime</scope> + </dependency> + </dependencies> + </profile> + <profile> + <id>hadoop2</id> + <properties> + <classifier>hadoop2</classifier> + </properties> + <dependencies> + <dependency> + <groupId>org.spark-project</groupId> + <artifactId>spark-core</artifactId> + <version>${project.version}</version> + <classifier>hadoop2</classifier> + </dependency> + <dependency> + <groupId>org.spark-project</groupId> + <artifactId>spark-bagel</artifactId> + <version>${project.version}</version> + <classifier>hadoop2</classifier> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>org.spark-project</groupId> + <artifactId>spark-examples</artifactId> + <version>${project.version}</version> + <classifier>hadoop2</classifier> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>org.spark-project</groupId> + <artifactId>spark-repl</artifactId> + <version>${project.version}</version> + <classifier>hadoop2</classifier> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-core</artifactId> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <scope>runtime</scope> + </dependency> + </dependencies> + </profile> + <profile> + <id>deb</id> + <build> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>buildnumber-maven-plugin</artifactId> + <version>1.1</version> + <executions> + <execution> + <phase>validate</phase> + <goals> + <goal>create</goal> + </goals> + <configuration> + <shortRevisionLength>8</shortRevisionLength> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.vafer</groupId> + <artifactId>jdeb</artifactId> + <version>0.11</version> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>jdeb</goal> + </goals> + <configuration> + <deb>${project.build.directory}/${project.artifactId}-${classifier}_${project.version}-${buildNumber}_all.deb</deb> + <attach>false</attach> + <compression>gzip</compression> + <dataSet> + <data> + <src>${project.build.directory}/${project.artifactId}-${project.version}-shaded-${classifier}.jar</src> + <type>file</type> + <mapper> + <type>perm</type> + <user>${deb.user}</user> + <group>${deb.user}</group> + <prefix>${deb.install.path}</prefix> + </mapper> + </data> + <data> + <src>${basedir}/src/deb/bin</src> + <type>directory</type> + <mapper> + <type>perm</type> + <user>${deb.user}</user> + <group>${deb.user}</group> + <prefix>${deb.install.path}</prefix> + <filemode>744</filemode> + </mapper> + </data> + <data> + <src>${basedir}/../conf</src> + <type>directory</type> + <mapper> + <type>perm</type> + <user>${deb.user}</user> + <group>${deb.user}</group> + <prefix>${deb.install.path}/conf</prefix> + <filemode>744</filemode> + </mapper> + </data> + </dataSet> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> +</project> diff --git a/repl/src/deb/bin/run b/repl-bin/src/deb/bin/run index c54c9e97a0..c54c9e97a0 100755 --- a/repl/src/deb/bin/run +++ b/repl-bin/src/deb/bin/run diff --git a/repl/src/deb/bin/spark-executor b/repl-bin/src/deb/bin/spark-executor index 47b9cccdfe..47b9cccdfe 100755 --- a/repl/src/deb/bin/spark-executor +++ b/repl-bin/src/deb/bin/spark-executor diff --git a/repl/src/deb/bin/spark-shell b/repl-bin/src/deb/bin/spark-shell index 219c66eb0b..219c66eb0b 100755 --- a/repl/src/deb/bin/spark-shell +++ b/repl-bin/src/deb/bin/spark-shell diff --git a/repl/src/deb/control/control b/repl-bin/src/deb/control/control index 6586986c76..afadb3fbfe 100644 --- a/repl/src/deb/control/control +++ b/repl-bin/src/deb/control/control @@ -1,4 +1,4 @@ -Package: spark-repl +Package: [[artifactId]] Version: [[version]]-[[buildNumber]] Section: misc Priority: extra diff --git a/repl/pom.xml b/repl/pom.xml index f6df4ba9f7..114e3e9932 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -99,46 +99,17 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-core</artifactId> + <scope>provided</scope> </dependency> </dependencies> <build> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> + <artifactId>maven-jar-plugin</artifactId> <configuration> - <shadedArtifactAttached>true</shadedArtifactAttached> - <shadedClassifierName>shaded-hadoop1</shadedClassifierName> - <filters> - <filter> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> + <classifier>hadoop1</classifier> </configuration> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> - <configuration> - <transformers> - <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> - <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer"> - <resource>reference.conf</resource> - </transformer> - <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> - <mainClass>spark.repl.Main</mainClass> - </transformer> - </transformers> - </configuration> - </execution> - </executions> </plugin> </plugins> </build> @@ -172,125 +143,22 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-core</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> + <scope>provided</scope> </dependency> </dependencies> <build> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> + <artifactId>maven-jar-plugin</artifactId> <configuration> - <shadedArtifactAttached>true</shadedArtifactAttached> - <shadedClassifierName>shaded-hadoop2</shadedClassifierName> - <filters> - <filter> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> + <classifier>hadoop2</classifier> </configuration> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> - <configuration> - <transformers> - <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> - <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer"> - <resource>reference.conf</resource> - </transformer> - <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> - <mainClass>spark.repl.Main</mainClass> - </transformer> - </transformers> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - <profile> - <id>deb</id> - <build> - <plugins> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>buildnumber-maven-plugin</artifactId> - <version>1.1</version> - <executions> - <execution> - <phase>validate</phase> - <goals> - <goal>create</goal> - </goals> - <configuration> - <shortRevisionLength>8</shortRevisionLength> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.vafer</groupId> - <artifactId>jdeb</artifactId> - <version>0.11</version> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>jdeb</goal> - </goals> - <configuration> - <deb>${project.build.directory}/${project.artifactId}-${classifier}_${project.version}-${buildNumber}_all.deb</deb> - <attach>false</attach> - <compression>gzip</compression> - <dataSet> - <data> - <src>${project.build.directory}/${project.artifactId}-${project.version}-shaded-${classifier}.jar</src> - <type>file</type> - <mapper> - <type>perm</type> - <user>${deb.user}</user> - <group>${deb.user}</group> - <prefix>${deb.install.path}</prefix> - </mapper> - </data> - <data> - <src>${basedir}/src/deb/bin</src> - <type>directory</type> - <mapper> - <type>perm</type> - <user>${deb.user}</user> - <group>${deb.user}</group> - <prefix>${deb.install.path}</prefix> - <filemode>744</filemode> - </mapper> - </data> - <data> - <src>${basedir}/../conf</src> - <type>directory</type> - <mapper> - <type>perm</type> - <user>${deb.user}</user> - <group>${deb.user}</group> - <prefix>${deb.install.path}/conf</prefix> - <filemode>744</filemode> - </mapper> - </data> - </dataSet> - </configuration> - </execution> - </executions> </plugin> </plugins> </build> |