diff options
author | Reynold Xin <rxin@apache.org> | 2014-09-04 23:34:58 -0700 |
---|---|---|
committer | Reynold Xin <rxin@apache.org> | 2014-09-04 23:34:58 -0700 |
commit | 1725a1a5d10a53762bd80f391eddbf306f2841ee (patch) | |
tree | 5644a7a34b5b113bfd6cdf981579a36cdf5c0d85 | |
parent | 1904bac38d97df5ae9fb193e92a83c7f8ff6d255 (diff) | |
download | spark-1725a1a5d10a53762bd80f391eddbf306f2841ee.tar.gz spark-1725a1a5d10a53762bd80f391eddbf306f2841ee.tar.bz2 spark-1725a1a5d10a53762bd80f391eddbf306f2841ee.zip |
[SPARK-3391][EC2] Support attaching up to 8 EBS volumes.
Please merge this at the same time as https://github.com/mesos/spark-ec2/pull/66
Author: Reynold Xin <rxin@apache.org>
Closes #2260 from rxin/ec2-ebs-vol and squashes the following commits:
b9527d9 [Reynold Xin] Removed io1 ebs type.
bf9c403 [Reynold Xin] Made EBS volume type configurable.
c8e25ea [Reynold Xin] Support up to 8 EBS volumes.
adf4f2e [Reynold Xin] Revert git repo change.
020c542 [Reynold Xin] [SPARK-3391] Support attaching more than 1 EBS volumes.
-rwxr-xr-x | ec2/spark_ec2.py | 33 |
1 files changed, 25 insertions, 8 deletions
diff --git a/ec2/spark_ec2.py b/ec2/spark_ec2.py index eed6eb8485..1670faca4a 100755 --- a/ec2/spark_ec2.py +++ b/ec2/spark_ec2.py @@ -102,9 +102,17 @@ def parse_args(): "(for debugging)") parser.add_option( "--ebs-vol-size", metavar="SIZE", type="int", default=0, - help="Attach a new EBS volume of size SIZE (in GB) to each node as " + - "/vol. The volumes will be deleted when the instances terminate. " + - "Only possible on EBS-backed AMIs.") + help="Size (in GB) of each EBS volume.") + parser.add_option( + "--ebs-vol-type", default="standard", + help="EBS volume type (e.g. 'gp2', 'standard').") + parser.add_option( + "--ebs-vol-num", type="int", default=1, + help="Number of EBS volumes to attach to each node as /vol[x]. " + + "The volumes will be deleted when the instances terminate. " + + "Only possible on EBS-backed AMIs. " + + "EBS volumes are only attached if --ebs-vol-size > 0." + + "Only support up to 8 EBS volumes.") parser.add_option( "--swap", metavar="SWAP", type="int", default=1024, help="Swap space to set up per node, in MB (default: 1024)") @@ -348,13 +356,16 @@ def launch_cluster(conn, opts, cluster_name): print >> stderr, "Could not find AMI " + opts.ami sys.exit(1) - # Create block device mapping so that we can add an EBS volume if asked to + # Create block device mapping so that we can add EBS volumes if asked to. + # The first drive is attached as /dev/sds, 2nd as /dev/sdt, ... /dev/sdz block_map = BlockDeviceMapping() if opts.ebs_vol_size > 0: - device = EBSBlockDeviceType() - device.size = opts.ebs_vol_size - device.delete_on_termination = True - block_map["/dev/sdv"] = device + for i in range(opts.ebs_vol_num): + device = EBSBlockDeviceType() + device.size = opts.ebs_vol_size + device.volume_type=opts.ebs_vol_type + device.delete_on_termination = True + block_map["/dev/sd" + chr(ord('s') + i)] = device # AWS ignores the AMI-specified block device mapping for M3 (see SPARK-3342). if opts.instance_type.startswith('m3.'): @@ -828,6 +839,12 @@ def get_partition(total, num_partitions, current_partitions): def real_main(): (opts, action, cluster_name) = parse_args() + + # Input parameter validation + if opts.ebs_vol_num > 8: + print >> stderr, "ebs-vol-num cannot be greater than 8" + sys.exit(1) + try: conn = ec2.connect_to_region(opts.region) except Exception as e: |