From d17c142615f7d0ff514a74779e433107659a78b9 Mon Sep 17 00:00:00 2001 From: Ewen Cheslack-Postava Date: Mon, 16 Dec 2013 08:09:37 -0800 Subject: Force pseudo-tty allocation in spark-ec2 script. ssh commands need the -t argument repeated twice if there is no local tty, e.g. if the process running spark-ec2 uses nohup and the parent process exits. --- ec2/spark_ec2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'ec2') diff --git a/ec2/spark_ec2.py b/ec2/spark_ec2.py index 1189232428..a2b0e7e7f4 100755 --- a/ec2/spark_ec2.py +++ b/ec2/spark_ec2.py @@ -589,7 +589,7 @@ def ssh(host, opts, command): while True: try: return subprocess.check_call( - ssh_command(opts) + ['-t', '%s@%s' % (opts.user, host), stringify_command(command)]) + ssh_command(opts) + ['-t', '-t', '%s@%s' % (opts.user, host), stringify_command(command)]) except subprocess.CalledProcessError as e: if (tries > 2): # If this was an ssh failure, provide the user with hints. @@ -730,7 +730,7 @@ def real_main(): if opts.proxy_port != None: proxy_opt = ['-D', opts.proxy_port] subprocess.check_call( - ssh_command(opts) + proxy_opt + ['-t', "%s@%s" % (opts.user, master)]) + ssh_command(opts) + proxy_opt + ['-t', '-t', "%s@%s" % (opts.user, master)]) elif action == "get-master": (master_nodes, slave_nodes) = get_existing_cluster(conn, opts, cluster_name) -- cgit v1.2.3 From 59e8009b8d5e51b6f776720de8c9ecb09e1072dc Mon Sep 17 00:00:00 2001 From: Prashant Sharma Date: Thu, 2 Jan 2014 18:54:08 +0530 Subject: a few left over document change --- docs/spark-standalone.md | 2 +- ec2/spark_ec2.py | 2 +- make-distribution.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'ec2') diff --git a/docs/spark-standalone.md b/docs/spark-standalone.md index ec5ae5b4f7..baa0a062f7 100644 --- a/docs/spark-standalone.md +++ b/docs/spark-standalone.md @@ -20,7 +20,7 @@ then modify `conf/spark-env.sh` in the `dist/` directory before deploying to all You can start a standalone master server by executing: - ./bin/start-master.sh + ./sbin/start-master.sh Once started, the master will print out a `spark://HOST:PORT` URL for itself, which you can use to connect workers to it, or pass as the "master" argument to `SparkContext`. You can also find this URL on diff --git a/ec2/spark_ec2.py b/ec2/spark_ec2.py index a2b0e7e7f4..d82a1e1490 100755 --- a/ec2/spark_ec2.py +++ b/ec2/spark_ec2.py @@ -436,7 +436,7 @@ def setup_cluster(conn, master_nodes, slave_nodes, opts, deploy_ssh_key): def setup_standalone_cluster(master, slave_nodes, opts): slave_ips = '\n'.join([i.public_dns_name for i in slave_nodes]) ssh(master, opts, "echo \"%s\" > spark/conf/slaves" % (slave_ips)) - ssh(master, opts, "/root/spark/bin/start-all.sh") + ssh(master, opts, "/root/spark/sbin/start-all.sh") def setup_spark_cluster(master, opts): ssh(master, opts, "chmod u+x spark-ec2/setup.sh") diff --git a/make-distribution.sh b/make-distribution.sh index 8765c7e620..0463d14762 100755 --- a/make-distribution.sh +++ b/make-distribution.sh @@ -31,9 +31,9 @@ # # Recommended deploy/testing procedure (standalone mode): # 1) Rsync / deploy the dist/ dir to one host -# 2) cd to deploy dir; ./bin/start-master.sh +# 2) cd to deploy dir; ./sbin/start-master.sh # 3) Verify master is up by visiting web page, ie http://master-ip:8080. Note the spark:// URL. -# 4) ./bin/start-slave.sh 1 <> +# 4) ./sbin/start-slave.sh 1 <> # 5) MASTER="spark://my-master-ip:7077" ./bin/spark-shell # -- cgit v1.2.3