From 256704c771d301700af9ebf0d180c1ba7c4116c0 Mon Sep 17 00:00:00 2001 From: Sean Owen Date: Wed, 9 Mar 2016 18:27:44 +0000 Subject: [SPARK-13595][BUILD] Move docker, extras modules into external ## What changes were proposed in this pull request? Move `docker` dirs out of top level into `external/`; move `extras/*` into `external/` ## How was this patch tested? This is tested with Jenkins tests. Author: Sean Owen Closes #11523 from srowen/SPARK-13595. --- docker/README.md | 7 ------- docker/build | 22 --------------------- docker/spark-mesos/Dockerfile | 30 ----------------------------- docker/spark-test/README.md | 11 ----------- docker/spark-test/base/Dockerfile | 37 ------------------------------------ docker/spark-test/build | 22 --------------------- docker/spark-test/master/Dockerfile | 21 -------------------- docker/spark-test/master/default_cmd | 28 --------------------------- docker/spark-test/worker/Dockerfile | 22 --------------------- docker/spark-test/worker/default_cmd | 28 --------------------------- 10 files changed, 228 deletions(-) delete mode 100644 docker/README.md delete mode 100755 docker/build delete mode 100644 docker/spark-mesos/Dockerfile delete mode 100644 docker/spark-test/README.md delete mode 100644 docker/spark-test/base/Dockerfile delete mode 100755 docker/spark-test/build delete mode 100644 docker/spark-test/master/Dockerfile delete mode 100755 docker/spark-test/master/default_cmd delete mode 100644 docker/spark-test/worker/Dockerfile delete mode 100755 docker/spark-test/worker/default_cmd (limited to 'docker') diff --git a/docker/README.md b/docker/README.md deleted file mode 100644 index 40ba9c3065..0000000000 --- a/docker/README.md +++ /dev/null @@ -1,7 +0,0 @@ -Spark docker files -=========== - -Drawn from Matt Massie's docker files (https://github.com/massie/dockerfiles), -as well as some updates from Andre Schumacher (https://github.com/AndreSchumacher/docker). - -Tested with Docker version 0.8.1. diff --git a/docker/build b/docker/build deleted file mode 100755 index 253a2fc8dd..0000000000 --- a/docker/build +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -docker images > /dev/null || { echo Please install docker in non-sudo mode. ; exit; } - -./spark-test/build \ No newline at end of file diff --git a/docker/spark-mesos/Dockerfile b/docker/spark-mesos/Dockerfile deleted file mode 100644 index fb3f267fe5..0000000000 --- a/docker/spark-mesos/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -# This is an example Dockerfile for creating a Spark image which can be -# references by the Spark property 'spark.mesos.executor.docker.image' -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM mesosphere/mesos:0.20.1 - -# Update the base ubuntu image with dependencies needed for Spark -RUN apt-get update && \ - apt-get install -y python libnss3 openjdk-7-jre-headless curl - -RUN mkdir /opt/spark && \ - curl http://www.apache.org/dyn/closer.lua/spark/spark-1.4.0/spark-1.4.0-bin-hadoop2.4.tgz \ - | tar -xzC /opt -ENV SPARK_HOME /opt/spark -ENV MESOS_NATIVE_JAVA_LIBRARY /usr/local/lib/libmesos.so diff --git a/docker/spark-test/README.md b/docker/spark-test/README.md deleted file mode 100644 index ec0baf6e6d..0000000000 --- a/docker/spark-test/README.md +++ /dev/null @@ -1,11 +0,0 @@ -Spark Docker files usable for testing and development purposes. - -These images are intended to be run like so: - - docker run -v $SPARK_HOME:/opt/spark spark-test-master - docker run -v $SPARK_HOME:/opt/spark spark-test-worker spark://:7077 - -Using this configuration, the containers will have their Spark directories -mounted to your actual `SPARK_HOME`, allowing you to modify and recompile -your Spark source and have them immediately usable in the docker images -(without rebuilding them). diff --git a/docker/spark-test/base/Dockerfile b/docker/spark-test/base/Dockerfile deleted file mode 100644 index 76f550f886..0000000000 --- a/docker/spark-test/base/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM ubuntu:precise - -# Upgrade package index -# install a few other useful packages plus Open Jdk 7 -# Remove unneeded /var/lib/apt/lists/* after install to reduce the -# docker image size (by ~30MB) -RUN apt-get update && \ - apt-get install -y less openjdk-7-jre-headless net-tools vim-tiny sudo openssh-server && \ - rm -rf /var/lib/apt/lists/* - -ENV SCALA_VERSION 2.11.7 -ENV CDH_VERSION cdh4 -ENV SCALA_HOME /opt/scala-$SCALA_VERSION -ENV SPARK_HOME /opt/spark -ENV PATH $SPARK_HOME:$SCALA_HOME/bin:$PATH - -# Install Scala -ADD http://www.scala-lang.org/files/archive/scala-$SCALA_VERSION.tgz / -RUN (cd / && gunzip < scala-$SCALA_VERSION.tgz)|(cd /opt && tar -xvf -) -RUN rm /scala-$SCALA_VERSION.tgz diff --git a/docker/spark-test/build b/docker/spark-test/build deleted file mode 100755 index 6f9e197433..0000000000 --- a/docker/spark-test/build +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -docker build -t spark-test-base spark-test/base/ -docker build -t spark-test-master spark-test/master/ -docker build -t spark-test-worker spark-test/worker/ diff --git a/docker/spark-test/master/Dockerfile b/docker/spark-test/master/Dockerfile deleted file mode 100644 index f729534ab6..0000000000 --- a/docker/spark-test/master/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -# Spark Master -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM spark-test-base -ADD default_cmd /root/ -CMD ["/root/default_cmd"] diff --git a/docker/spark-test/master/default_cmd b/docker/spark-test/master/default_cmd deleted file mode 100755 index 5a7da3446f..0000000000 --- a/docker/spark-test/master/default_cmd +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -IP=$(ip -o -4 addr list eth0 | perl -n -e 'if (m{inet\s([\d\.]+)\/\d+\s}xms) { print $1 }') -echo "CONTAINER_IP=$IP" -export SPARK_LOCAL_IP=$IP -export SPARK_PUBLIC_DNS=$IP - -# Avoid the default Docker behavior of mapping our IP address to an unreachable host name -umount /etc/hosts - -/opt/spark/bin/spark-class org.apache.spark.deploy.master.Master -i $IP diff --git a/docker/spark-test/worker/Dockerfile b/docker/spark-test/worker/Dockerfile deleted file mode 100644 index 890febe7b6..0000000000 --- a/docker/spark-test/worker/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -# Spark Worker -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM spark-test-base -ENV SPARK_WORKER_PORT 8888 -ADD default_cmd /root/ -ENTRYPOINT ["/root/default_cmd"] diff --git a/docker/spark-test/worker/default_cmd b/docker/spark-test/worker/default_cmd deleted file mode 100755 index 31b06cb0eb..0000000000 --- a/docker/spark-test/worker/default_cmd +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -IP=$(ip -o -4 addr list eth0 | perl -n -e 'if (m{inet\s([\d\.]+)\/\d+\s}xms) { print $1 }') -echo "CONTAINER_IP=$IP" -export SPARK_LOCAL_IP=$IP -export SPARK_PUBLIC_DNS=$IP - -# Avoid the default Docker behavior of mapping our IP address to an unreachable host name -umount /etc/hosts - -/opt/spark/bin/spark-class org.apache.spark.deploy.worker.Worker $1 -- cgit v1.2.3