diff options
author | Stefano Parmesan <s.parmesan@gmail.com> | 2015-06-22 11:43:10 -0700 |
---|---|---|
committer | Shivaram Venkataraman <shivaram@cs.berkeley.edu> | 2015-06-22 11:43:10 -0700 |
commit | 42a1f716fa35533507784be5e9117a984a03e62d (patch) | |
tree | 7f65486e6b1b08b7bd7e7e2e55e347c2395ae3d4 | |
parent | 0818fdec3733ec5c0a9caa48a9c0f2cd25f84d13 (diff) | |
download | spark-42a1f716fa35533507784be5e9117a984a03e62d.tar.gz spark-42a1f716fa35533507784be5e9117a984a03e62d.tar.bz2 spark-42a1f716fa35533507784be5e9117a984a03e62d.zip |
[SPARK-8429] [EC2] Add ability to set additional tags
Add the `--additional-tags` parameter that allows to set additional tags to all the created instances (masters and slaves).
The user can specify multiple tags by separating them with a comma (`,`), while each tag name and value should be separated by a colon (`:`); for example, `Task:MySparkProject,Env:production` would add two tags, `Task` and `Env`, with the given values.
Author: Stefano Parmesan <s.parmesan@gmail.com>
Closes #6857 from armisael/patch-1 and squashes the following commits:
c5ac92c [Stefano Parmesan] python style (pep8)
8e614f1 [Stefano Parmesan] Set multiple tags in a single request
bfc56af [Stefano Parmesan] Address SPARK-7900 by inceasing sleep time
daf8615 [Stefano Parmesan] Add ability to set additional tags
-rwxr-xr-x | ec2/spark_ec2.py | 28 |
1 files changed, 20 insertions, 8 deletions
diff --git a/ec2/spark_ec2.py b/ec2/spark_ec2.py index 5608749946..1037356854 100755 --- a/ec2/spark_ec2.py +++ b/ec2/spark_ec2.py @@ -290,6 +290,10 @@ def parse_args(): "--additional-security-group", type="string", default="", help="Additional security group to place the machines in") parser.add_option( + "--additional-tags", type="string", default="", + help="Additional tags to set on the machines; tags are comma-separated, while name and " + + "value are colon separated; ex: \"Task:MySparkProject,Env:production\"") + parser.add_option( "--copy-aws-credentials", action="store_true", default=False, help="Add AWS credentials to hadoop configuration to allow Spark to access S3") parser.add_option( @@ -684,16 +688,24 @@ def launch_cluster(conn, opts, cluster_name): # This wait time corresponds to SPARK-4983 print("Waiting for AWS to propagate instance metadata...") - time.sleep(5) - # Give the instances descriptive names + time.sleep(15) + + # Give the instances descriptive names and set additional tags + additional_tags = {} + if opts.additional_tags.strip(): + additional_tags = dict( + map(str.strip, tag.split(':', 1)) for tag in opts.additional_tags.split(',') + ) + for master in master_nodes: - master.add_tag( - key='Name', - value='{cn}-master-{iid}'.format(cn=cluster_name, iid=master.id)) + master.add_tags( + dict(additional_tags, Name='{cn}-master-{iid}'.format(cn=cluster_name, iid=master.id)) + ) + for slave in slave_nodes: - slave.add_tag( - key='Name', - value='{cn}-slave-{iid}'.format(cn=cluster_name, iid=slave.id)) + slave.add_tags( + dict(additional_tags, Name='{cn}-slave-{iid}'.format(cn=cluster_name, iid=slave.id)) + ) # Return all the instances return (master_nodes, slave_nodes) |