diff options
author | cheng chang <myairia@gmail.com> | 2015-03-10 11:02:12 +0000 |
---|---|---|
committer | Sean Owen <sowen@cloudera.com> | 2015-03-10 11:02:54 +0000 |
commit | 7c7d2d5e093b0623edc75dd166ec1179b4e62062 (patch) | |
tree | 88b370cc6b5b6edb854dd7fd44571bfa91618927 /ec2/spark_ec2.py | |
parent | d14df06c05a6228fd6522914c39aa75898eddfc1 (diff) | |
download | spark-7c7d2d5e093b0623edc75dd166ec1179b4e62062.tar.gz spark-7c7d2d5e093b0623edc75dd166ec1179b4e62062.tar.bz2 spark-7c7d2d5e093b0623edc75dd166ec1179b4e62062.zip |
[SPARK-6186] [EC2] Make Tachyon version configurable in EC2 deployment script
This PR comes from Tachyon community to solve the issue:
https://tachyon.atlassian.net/browse/TACHYON-11
An accompanying PR is in mesos/spark-ec2:
https://github.com/mesos/spark-ec2/pull/101
Author: cheng chang <myairia@gmail.com>
Closes #4901 from uronce-cc/master and squashes the following commits:
313aa36 [cheng chang] minor re-wording
fd2a48e [cheng chang] Remove Tachyon when deploying through git hash
1d53c5c [cheng chang] add default value to --tachyon-version
6f8887e [cheng chang] make tachyon version configurable
Diffstat (limited to 'ec2/spark_ec2.py')
-rwxr-xr-x | ec2/spark_ec2.py | 19 |
1 files changed, 19 insertions, 0 deletions
diff --git a/ec2/spark_ec2.py b/ec2/spark_ec2.py index 3acb5fea04..f848874b0c 100755 --- a/ec2/spark_ec2.py +++ b/ec2/spark_ec2.py @@ -62,6 +62,16 @@ VALID_SPARK_VERSIONS = set([ "1.2.1", ]) +SPARK_TACHYON_MAP = { + "1.0.0": "0.4.1", + "1.0.1": "0.4.1", + "1.0.2": "0.4.1", + "1.1.0": "0.5.0", + "1.1.1": "0.5.0", + "1.2.0": "0.5.0", + "1.2.1": "0.5.0", +} + DEFAULT_SPARK_VERSION = SPARK_EC2_VERSION DEFAULT_SPARK_GITHUB_REPO = "https://github.com/apache/spark" @@ -370,6 +380,10 @@ EC2_INSTANCE_TYPES = { } +def get_tachyon_version(spark_version): + return SPARK_TACHYON_MAP.get(spark_version, "") + + # Attempt to resolve an appropriate AMI given the architecture and region of the request. def get_spark_ami(opts): if opts.instance_type in EC2_INSTANCE_TYPES: @@ -919,9 +933,13 @@ def deploy_files(conn, root_dir, opts, master_nodes, slave_nodes, modules): if "." in opts.spark_version: # Pre-built Spark deploy spark_v = get_validate_spark_version(opts.spark_version, opts.spark_git_repo) + tachyon_v = get_tachyon_version(spark_v) else: # Spark-only custom deploy spark_v = "%s|%s" % (opts.spark_git_repo, opts.spark_version) + tachyon_v = "" + print "Deploying Spark via git hash; Tachyon won't be set up" + modules = filter(lambda x: x != "tachyon", modules) template_vars = { "master_list": '\n'.join([i.public_dns_name for i in master_nodes]), @@ -934,6 +952,7 @@ def deploy_files(conn, root_dir, opts, master_nodes, slave_nodes, modules): "swap": str(opts.swap), "modules": '\n'.join(modules), "spark_version": spark_v, + "tachyon_version": tachyon_v, "hadoop_major_version": opts.hadoop_major_version, "spark_worker_instances": "%d" % opts.worker_instances, "spark_master_opts": opts.master_opts |