diff options
author | Aaron Davidson <aaron@databricks.com> | 2013-10-05 23:07:26 -0700 |
---|---|---|
committer | Aaron Davidson <aaron@databricks.com> | 2013-10-05 23:20:31 -0700 |
commit | e1190229e13453cdb1e7c28fdf300d1f8dd717c2 (patch) | |
tree | 25a210aadb5fc32c9104b9415566d030a94c28d5 /docker | |
parent | 0f070279e7cd224de48333b572d3080b742a82d7 (diff) | |
download | spark-e1190229e13453cdb1e7c28fdf300d1f8dd717c2.tar.gz spark-e1190229e13453cdb1e7c28fdf300d1f8dd717c2.tar.bz2 spark-e1190229e13453cdb1e7c28fdf300d1f8dd717c2.zip |
Add end-to-end test for standalone scheduler fault tolerance
Docker files drawn mostly from Matt Masse. Some updates from Andre Schumacher.
Diffstat (limited to 'docker')
-rw-r--r-- | docker/README.md | 5 | ||||
-rwxr-xr-x | docker/build | 5 | ||||
-rw-r--r-- | docker/spark-test/README.md | 10 | ||||
-rw-r--r-- | docker/spark-test/base/Dockerfile | 23 | ||||
-rwxr-xr-x | docker/spark-test/build | 5 | ||||
-rw-r--r-- | docker/spark-test/master/Dockerfile | 4 | ||||
-rwxr-xr-x | docker/spark-test/master/default_cmd | 4 | ||||
-rw-r--r-- | docker/spark-test/worker/Dockerfile | 5 | ||||
-rwxr-xr-x | docker/spark-test/worker/default_cmd | 4 |
9 files changed, 65 insertions, 0 deletions
diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000000..bf59e77d11 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,5 @@ +Spark docker files +=========== + +Drawn from Matt Massie's docker files (https://github.com/massie/dockerfiles), +as well as some updates from Andre Schumacher (https://github.com/AndreSchumacher/docker).
\ No newline at end of file diff --git a/docker/build b/docker/build new file mode 100755 index 0000000000..bb2d972cd2 --- /dev/null +++ b/docker/build @@ -0,0 +1,5 @@ +#!/bin/bash + +docker images > /dev/null || { echo Please install docker in non-sudo mode. ; exit; } + +./spark-test/build
\ No newline at end of file diff --git a/docker/spark-test/README.md b/docker/spark-test/README.md new file mode 100644 index 0000000000..addea277aa --- /dev/null +++ b/docker/spark-test/README.md @@ -0,0 +1,10 @@ +Spark Docker files usable for testing and development purposes. + +These images are intended to be run like so: +docker run -v $SPARK_HOME:/opt/spark spark-test-master +docker run -v $SPARK_HOME:/opt/spark spark-test-worker <master_ip> + +Using this configuration, the containers will have their Spark directories +mounted to your actual SPARK_HOME, allowing you to modify and recompile +your Spark source and have them immediately usable in the docker images +(without rebuilding them). diff --git a/docker/spark-test/base/Dockerfile b/docker/spark-test/base/Dockerfile new file mode 100644 index 0000000000..f9e7217af1 --- /dev/null +++ b/docker/spark-test/base/Dockerfile @@ -0,0 +1,23 @@ +# Spark 0.8.1 +# +FROM ubuntu:precise + +RUN echo "deb http://archive.ubuntu.com/ubuntu precise main universe" > /etc/apt/sources.list + +# Upgrade package index +RUN apt-get update + +# install a few other useful packages plus Open Jdk 7 +RUN apt-get install -y less openjdk-7-jre-headless net-tools vim-tiny sudo openssh-server + +ENV SCALA_VERSION 2.9.3 +ENV SPARK_VERSION 0.8.1 +ENV CDH_VERSION cdh4 +ENV SCALA_HOME /opt/scala-$SCALA_VERSION +ENV SPARK_HOME /opt/spark +ENV PATH $SPARK_HOME:$SCALA_HOME/bin:$PATH + +# Install Scala +ADD http://www.scala-lang.org/files/archive/scala-$SCALA_VERSION.tgz / +RUN (cd / && gunzip < scala-$SCALA_VERSION.tgz)|(cd /opt && tar -xvf -) +RUN rm /scala-$SCALA_VERSION.tgz diff --git a/docker/spark-test/build b/docker/spark-test/build new file mode 100755 index 0000000000..3835fcf279 --- /dev/null +++ b/docker/spark-test/build @@ -0,0 +1,5 @@ +#!/bin/bash + +docker build -t spark-test-base spark-test/base/ +docker build -t spark-test-master spark-test/master/ +docker build -t spark-test-worker spark-test/worker/ diff --git a/docker/spark-test/master/Dockerfile b/docker/spark-test/master/Dockerfile new file mode 100644 index 0000000000..9e6ddb1abc --- /dev/null +++ b/docker/spark-test/master/Dockerfile @@ -0,0 +1,4 @@ +# Spark Master +FROM spark-test-base +ADD default_cmd /root/ +CMD ["/root/default_cmd"] diff --git a/docker/spark-test/master/default_cmd b/docker/spark-test/master/default_cmd new file mode 100755 index 0000000000..92910068a3 --- /dev/null +++ b/docker/spark-test/master/default_cmd @@ -0,0 +1,4 @@ +#!/bin/bash +IP=$(ip -o -4 addr list eth0 | perl -n -e 'if (m{inet\s([\d\.]+)\/\d+\s}xms) { print $1 }') +echo "CONTAINER_IP=$IP" +/opt/spark/spark-class org.apache.spark.deploy.master.Master -i $IP diff --git a/docker/spark-test/worker/Dockerfile b/docker/spark-test/worker/Dockerfile new file mode 100644 index 0000000000..5690678051 --- /dev/null +++ b/docker/spark-test/worker/Dockerfile @@ -0,0 +1,5 @@ +# Spark Worker +FROM spark-test-base +ENV SPARK_WORKER_PORT 8888 +ADD default_cmd /root/ +ENTRYPOINT ["/root/default_cmd"] diff --git a/docker/spark-test/worker/default_cmd b/docker/spark-test/worker/default_cmd new file mode 100755 index 0000000000..360993f17c --- /dev/null +++ b/docker/spark-test/worker/default_cmd @@ -0,0 +1,4 @@ +#!/bin/bash +IP=$(ip -o -4 addr list eth0 | perl -n -e 'if (m{inet\s([\d\.]+)\/\d+\s}xms) { print $1 }') +echo "CONTAINER_IP=$IP" +/opt/spark/spark-class org.apache.spark.deploy.worker.Worker $1 |