diff options
author | Reza Zadeh <rizlar@gmail.com> | 2014-03-20 10:39:20 -0700 |
---|---|---|
committer | Matei Zaharia <matei@databricks.com> | 2014-03-20 10:39:20 -0700 |
commit | 66a03e5fe0167f590d150e099b15902e826a188f (patch) | |
tree | dcefdd0a43f1fdb80b13a929ec13aec96d4682cb /examples/src/main/scala | |
parent | ffe272d97c22955fe7744b1c0132cd9877b6df96 (diff) | |
download | spark-66a03e5fe0167f590d150e099b15902e826a188f.tar.gz spark-66a03e5fe0167f590d150e099b15902e826a188f.tar.bz2 spark-66a03e5fe0167f590d150e099b15902e826a188f.zip |
Principal Component Analysis
# Principal Component Analysis
Computes the top k principal component coefficients for the m-by-n data matrix X. Rows of X correspond to observations and columns correspond to variables. The coefficient matrix is n-by-k. Each column of the coefficients return matrix contains coefficients for one principal component, and the columns are in descending order of component variance. This function centers the data and uses the singular value decomposition (SVD) algorithm.
## Testing
Tests included:
* All principal components
* Only top k principal components
* Dense SVD tests
* Dense/sparse matrix tests
The results are tested against MATLAB's pca: http://www.mathworks.com/help/stats/pca.html
## Documentation
Added to mllib-guide.md
## Example Usage
Added to examples directory under SparkPCA.scala
Author: Reza Zadeh <rizlar@gmail.com>
Closes #88 from rezazadeh/sparkpca and squashes the following commits:
e298700 [Reza Zadeh] reformat using IDE
3f23271 [Reza Zadeh] documentation and cleanup
b025ab2 [Reza Zadeh] documentation
e2667d4 [Reza Zadeh] assertMatrixApproximatelyEquals
3787bb4 [Reza Zadeh] stylin
c6ecc1f [Reza Zadeh] docs
aa2bbcb [Reza Zadeh] rename sparseToTallSkinnyDense
56975b0 [Reza Zadeh] docs
2df9bde [Reza Zadeh] docs update
8fb0015 [Reza Zadeh] rcond documentation
dbf7797 [Reza Zadeh] correct argument number
a9f1f62 [Reza Zadeh] documentation
4ce6caa [Reza Zadeh] style changes
9a56a02 [Reza Zadeh] use rcond relative to larget svalue
120f796 [Reza Zadeh] housekeeping
156ff78 [Reza Zadeh] string comprehension
2e1cf43 [Reza Zadeh] rename rcond
ea223a6 [Reza Zadeh] many style changes
f4002d7 [Reza Zadeh] more docs
bd53c7a [Reza Zadeh] proper accumulator
a8b5ecf [Reza Zadeh] Don't use for loops
0dc7980 [Reza Zadeh] filter zeros in sparse
6115610 [Reza Zadeh] More documentation
36d51e8 [Reza Zadeh] use JBLAS for UVS^-1 computation
bc4599f [Reza Zadeh] configurable rcond
86f7515 [Reza Zadeh] compute per parition, use while
09726b3 [Reza Zadeh] more style changes
4195e69 [Reza Zadeh] private, accumulator
17002be [Reza Zadeh] style changes
4ba7471 [Reza Zadeh] style change
f4982e6 [Reza Zadeh] Use dense matrix in example
2828d28 [Reza Zadeh] optimizations: normalize once, use inplace ops
72c9fa1 [Reza Zadeh] rename DenseMatrix to TallSkinnyDenseMatrix, lean
f807be9 [Reza Zadeh] fix typo
2d7ccde [Reza Zadeh] Array interface for dense svd and pca
cd290fa [Reza Zadeh] provide RDD[Array[Double]] support
398d123 [Reza Zadeh] style change
55abbfa [Reza Zadeh] docs fix
ef29644 [Reza Zadeh] bad chnage undo
472566e [Reza Zadeh] all files from old pr
555168f [Reza Zadeh] initial files
Diffstat (limited to 'examples/src/main/scala')
-rw-r--r-- | examples/src/main/scala/org/apache/spark/examples/mllib/SparkPCA.scala | 51 | ||||
-rw-r--r-- | examples/src/main/scala/org/apache/spark/examples/mllib/SparkSVD.scala | 4 |
2 files changed, 53 insertions, 2 deletions
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/SparkPCA.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/SparkPCA.scala new file mode 100644 index 0000000000..d4e08c5e12 --- /dev/null +++ b/examples/src/main/scala/org/apache/spark/examples/mllib/SparkPCA.scala @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.examples.mllib + +import org.apache.spark.SparkContext +import org.apache.spark.mllib.linalg.PCA +import org.apache.spark.mllib.linalg.MatrixEntry +import org.apache.spark.mllib.linalg.SparseMatrix +import org.apache.spark.mllib.util._ + + +/** + * Compute PCA of an example matrix. + */ +object SparkPCA { + def main(args: Array[String]) { + if (args.length != 3) { + System.err.println("Usage: SparkPCA <master> m n") + System.exit(1) + } + val sc = new SparkContext(args(0), "PCA", + System.getenv("SPARK_HOME"), SparkContext.jarOfClass(this.getClass)) + + val m = args(2).toInt + val n = args(3).toInt + + // Make example matrix + val data = Array.tabulate(m, n) { (a, b) => + (a + 2).toDouble * (b + 1) / (1 + a + b) } + + // recover top principal component + val coeffs = new PCA().setK(1).compute(sc.makeRDD(data)) + + println("top principal component = " + coeffs.mkString(", ")) + } +} diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/SparkSVD.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/SparkSVD.scala index ce2b133368..2933cec497 100644 --- a/examples/src/main/scala/org/apache/spark/examples/mllib/SparkSVD.scala +++ b/examples/src/main/scala/org/apache/spark/examples/mllib/SparkSVD.scala @@ -38,7 +38,7 @@ object SparkSVD { System.exit(1) } val sc = new SparkContext(args(0), "SVD", - System.getenv("SPARK_HOME"), Seq(System.getenv("SPARK_EXAMPLES_JAR"))) + System.getenv("SPARK_HOME"), SparkContext.jarOfClass(this.getClass)) // Load and parse the data file val data = sc.textFile(args(1)).map { line => @@ -49,7 +49,7 @@ object SparkSVD { val n = args(3).toInt // recover largest singular vector - val decomposed = SVD.sparseSVD(SparseMatrix(data, m, n), 1) + val decomposed = new SVD().setK(1).compute(SparseMatrix(data, m, n)) val u = decomposed.U.data val s = decomposed.S.data val v = decomposed.V.data |