diff options
author | Aaron Davidson <aaron@databricks.com> | 2013-09-19 14:09:26 -0700 |
---|---|---|
committer | Aaron Davidson <aaron@databricks.com> | 2013-09-19 14:09:26 -0700 |
commit | f589ce771a53dcbfda5f62fe0ac77164688ecc76 (patch) | |
tree | 683793146c5449f65b8d3efd9b2b6704df3c4960 | |
parent | 2aff7989ab617f33052098498119886c40794774 (diff) | |
download | spark-f589ce771a53dcbfda5f62fe0ac77164688ecc76.tar.gz spark-f589ce771a53dcbfda5f62fe0ac77164688ecc76.tar.bz2 spark-f589ce771a53dcbfda5f62fe0ac77164688ecc76.zip |
Fix issue with spark_ec2 seeing empty security groups
Under unknown, but occasional, circumstances, reservation.groups is empty
despite reservation.instances each having groups. This means that the
spark_ec2 get_existing_clusters() method would fail to find any instances.
To fix it, we simply use the instances' groups as the source of truth.
Note that this is actually just a revival of PR #827, now that the issue
has been reproduced.
-rwxr-xr-x | ec2/spark_ec2.py | 8 |
1 files changed, 4 insertions, 4 deletions
diff --git a/ec2/spark_ec2.py b/ec2/spark_ec2.py index 419d0fe13f..6b7d202a88 100755 --- a/ec2/spark_ec2.py +++ b/ec2/spark_ec2.py @@ -364,12 +364,12 @@ def get_existing_cluster(conn, opts, cluster_name, die_on_error=True): slave_nodes = [] for res in reservations: active = [i for i in res.instances if is_active(i)] - if len(active) > 0: - group_names = [g.name for g in res.groups] + for inst in active: + group_names = [g.name for g in inst.groups] if group_names == [cluster_name + "-master"]: - master_nodes += res.instances + master_nodes.append(inst) elif group_names == [cluster_name + "-slaves"]: - slave_nodes += res.instances + slave_nodes.append(inst) if any((master_nodes, slave_nodes)): print ("Found %d master(s), %d slaves" % (len(master_nodes), len(slave_nodes))) |