aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--mllib/src/main/scala/org/apache/spark/mllib/api/python/PythonMLLibAPI.scala10
-rw-r--r--python/pyspark/mllib/feature.py35
2 files changed, 45 insertions, 0 deletions
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/api/python/PythonMLLibAPI.scala b/mllib/src/main/scala/org/apache/spark/mllib/api/python/PythonMLLibAPI.scala
index 2897865af6..634d56d08d 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/api/python/PythonMLLibAPI.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/api/python/PythonMLLibAPI.scala
@@ -520,6 +520,16 @@ private[python] class PythonMLLibAPI extends Serializable {
}
/**
+ * Java stub for PCA.fit(). This stub returns a
+ * handle to the Java object instead of the content of the Java object.
+ * Extra care needs to be taken in the Python code to ensure it gets freed on
+ * exit; see the Py4J documentation.
+ */
+ def fitPCA(k: Int, data: JavaRDD[Vector]): PCAModel = {
+ new PCA(k).fit(data.rdd)
+ }
+
+ /**
* Java stub for IDF.fit(). This stub returns a
* handle to the Java object instead of the content of the Java object.
* Extra care needs to be taken in the Python code to ensure it gets freed on
diff --git a/python/pyspark/mllib/feature.py b/python/pyspark/mllib/feature.py
index cf5fdf2cf9..334f5b86cd 100644
--- a/python/pyspark/mllib/feature.py
+++ b/python/pyspark/mllib/feature.py
@@ -252,6 +252,41 @@ class ChiSqSelector(object):
return ChiSqSelectorModel(jmodel)
+class PCAModel(JavaVectorTransformer):
+ """
+ Model fitted by [[PCA]] that can project vectors to a low-dimensional space using PCA.
+ """
+
+
+class PCA(object):
+ """
+ A feature transformer that projects vectors to a low-dimensional space using PCA.
+
+ >>> data = [Vectors.sparse(5, [(1, 1.0), (3, 7.0)]),
+ ... Vectors.dense([2.0, 0.0, 3.0, 4.0, 5.0]),
+ ... Vectors.dense([4.0, 0.0, 0.0, 6.0, 7.0])]
+ >>> model = PCA(2).fit(sc.parallelize(data))
+ >>> pcArray = model.transform(Vectors.sparse(5, [(1, 1.0), (3, 7.0)])).toArray()
+ >>> pcArray[0]
+ 1.648...
+ >>> pcArray[1]
+ -4.013...
+ """
+ def __init__(self, k):
+ """
+ :param k: number of principal components.
+ """
+ self.k = int(k)
+
+ def fit(self, data):
+ """
+ Computes a [[PCAModel]] that contains the principal components of the input vectors.
+ :param data: source vectors
+ """
+ jmodel = callMLlibFunc("fitPCA", self.k, data)
+ return PCAModel(jmodel)
+
+
class HashingTF(object):
"""
.. note:: Experimental