diff options
| author | Patrick Wendell <pwendell@gmail.com> | 2013-06-12 18:39:01 -0700 |
|---|---|---|
| committer | Patrick Wendell <pwendell@gmail.com> | 2013-06-12 18:39:01 -0700 |
| commit | eae7590d0d7c3093bcedefdf2ec4f75d7a8c62b8 (patch) | |
| tree | 450d7348172bde4482b7ae56fa3db540c8db401b /ec2 | |
| parent | bb328c8ff8aeb74c4ecc38a30bbb85f1c550ac3d (diff) | |
| download | spark-eae7590d0d7c3093bcedefdf2ec4f75d7a8c62b8.tar.gz spark-eae7590d0d7c3093bcedefdf2ec4f75d7a8c62b8.tar.bz2 spark-eae7590d0d7c3093bcedefdf2ec4f75d7a8c62b8.zip | |
Support for multiple hadoop versions
Diffstat (limited to 'ec2')
| -rw-r--r-- | ec2/deploy.generic/root/spark-ec2/ec2-variables.sh | 1 | ||||
| -rwxr-xr-x | ec2/spark_ec2.py | 5 |
2 files changed, 5 insertions, 1 deletions
diff --git a/ec2/deploy.generic/root/spark-ec2/ec2-variables.sh b/ec2/deploy.generic/root/spark-ec2/ec2-variables.sh index 453712bd25..72bf911a3f 100644 --- a/ec2/deploy.generic/root/spark-ec2/ec2-variables.sh +++ b/ec2/deploy.generic/root/spark-ec2/ec2-variables.sh @@ -10,4 +10,5 @@ export MESOS_SPARK_LOCAL_DIRS="{{spark_local_dirs}}" export MODULES="{{modules}}" export SPARK_VERSION="{{spark_version}}" export SHARK_VERSION="{{shark_version}}" +export HADOOP_MAJOR_VERSION="{{hadoop_major_version}}" export SWAP_MB="{{swap}}" diff --git a/ec2/spark_ec2.py b/ec2/spark_ec2.py index 5c728fb846..7d2de3e7d2 100755 --- a/ec2/spark_ec2.py +++ b/ec2/spark_ec2.py @@ -76,6 +76,8 @@ def parse_args(): parser.add_option("--shark-git-repo", default="https://github.com/amplab/shark", help="Github repo from which to checkout supplied commit hash") + parser.add_option("--hadoop-major-version", default="2", + help="Major version of Hadoop (default: 2)") parser.add_option("-D", metavar="[ADDRESS:]PORT", dest="proxy_port", help="Use SSH dynamic port forwarding to create a SOCKS proxy at " + @@ -565,7 +567,8 @@ def deploy_files(conn, root_dir, opts, master_nodes, slave_nodes, zoo_nodes, "swap": str(opts.swap), "modules": '\n'.join(modules), "spark_version": spark_v, - "shark_version": shark_v + "shark_version": shark_v, + "hadoop_major_version": opts.hadoop_major_version } # Create a temp directory in which we will place all the files to be |