diff options
author | Reza Zadeh <rizlar@gmail.com> | 2013-12-27 01:51:19 -0500 |
---|---|---|
committer | Reza Zadeh <rizlar@gmail.com> | 2013-12-27 01:51:19 -0500 |
commit | 642ab5c1e1ba98833265447447702c3c39fb2d40 (patch) | |
tree | c058583cc93d78823422f360035e9124d65f98dd | |
parent | 3369c2d48795d831acd841b8ecb67b5a84083883 (diff) | |
download | spark-642ab5c1e1ba98833265447447702c3c39fb2d40.tar.gz spark-642ab5c1e1ba98833265447447702c3c39fb2d40.tar.bz2 spark-642ab5c1e1ba98833265447447702c3c39fb2d40.zip |
initial large scale testing begin
-rwxr-xr-x | ec2/spark_ec2.py | 12 | ||||
-rw-r--r-- | mllib/src/main/scala/org/apache/spark/mllib/linalg/sparsesvd.scala | 8 |
2 files changed, 5 insertions, 15 deletions
diff --git a/ec2/spark_ec2.py b/ec2/spark_ec2.py index a2b0e7e7f4..5e8b381a4d 100755 --- a/ec2/spark_ec2.py +++ b/ec2/spark_ec2.py @@ -113,16 +113,6 @@ def parse_args(): # Boto config check # http://boto.cloudhackers.com/en/latest/boto_config_tut.html home_dir = os.getenv('HOME') - if home_dir == None or not os.path.isfile(home_dir + '/.boto'): - if not os.path.isfile('/etc/boto.cfg'): - if os.getenv('AWS_ACCESS_KEY_ID') == None: - print >> stderr, ("ERROR: The environment variable AWS_ACCESS_KEY_ID " + - "must be set") - sys.exit(1) - if os.getenv('AWS_SECRET_ACCESS_KEY') == None: - print >> stderr, ("ERROR: The environment variable AWS_SECRET_ACCESS_KEY " + - "must be set") - sys.exit(1) return (opts, action, cluster_name) @@ -646,7 +636,7 @@ def get_partition(total, num_partitions, current_partitions): def real_main(): (opts, action, cluster_name) = parse_args() try: - conn = ec2.connect_to_region(opts.region) + conn = ec2.connect_to_region(opts.region,aws_access_key_id="AKIAI2EGAQ7GYNL4LRAA", aws_secret_access_key="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") except Exception as e: print >> stderr, (e) sys.exit(1) diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/sparsesvd.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/sparsesvd.scala index 1c9f67e265..edf715dc19 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/sparsesvd.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/sparsesvd.scala @@ -32,8 +32,8 @@ import org.jblas.{DoubleMatrix, Singular, MatrixFunctions} * There is no restriction on m, but we require n^2 doubles to fit in memory. * Further, n should be less than m. * - * This is computed by first computing A'A = V S^2 V', - * computing locally on that (since n x n is small), + * The decomposition is computed by first computing A'A = V S^2 V', + * computing svd locally on that (since n x n is small), * from which we recover S and V. * Then we compute U via easy matrix multiplication * as U = A * V * S^-1 @@ -43,8 +43,8 @@ import org.jblas.{DoubleMatrix, Singular, MatrixFunctions} * such values, then the dimensions of the return will be: * * S is k x k and diagonal, holding the singular values on diagonal - * U is m x k and satisfies U'U = eye(k,k) - * V is n x k and satisfies V'V = eye(k,k) + * U is m x k and satisfies U'U = eye(k) + * V is n x k and satisfies V'V = eye(k) * * All input and output is expected in sparse matrix format, 1-indexed * as tuples of the form ((i,j),value) all in RDDs |