[SPARK-9918] [MLLIB] remove runs from k-means and rename epsilon to tol

This requires some discussion. I'm not sure whether `runs` is a useful parameter. It certainly complicates the implementation. We might want to optimize the k-means implementation with block matrix operations. In this case, having `runs` may not be worth the trade-off. Also it increases the communication cost in a single job, which might cause other issues.

This PR also renames `epsilon` to `tol` to have consistent naming among algorithms. The Python constructor is updated to include all parameters.

jkbradley yu-iskw

Author: Xiangrui Meng <meng@databricks.com>

Closes #8148 from mengxr/SPARK-9918 and squashes the following commits:

149b9e5 [Xiangrui Meng] fix constructor in Python and rename epsilon to tol
3cc15b3 [Xiangrui Meng] fix test and change initStep to initSteps in python
a0a0274 [Xiangrui Meng] remove runs from k-means in the pipeline API

1 files changed, 13 insertions, 50 deletions

diff --git a/python/pyspark/ml/clustering.py b/python/pyspark/ml/clustering.py
index 48338713a2..cb4c16e25a 100644
--- a/python/pyspark/ml/clustering.py
+++ b/python/pyspark/ml/clustering.py
@@ -19,7 +19,6 @@ from pyspark.ml.util import keyword_only
 from pyspark.ml.wrapper import JavaEstimator, JavaModel
 from pyspark.ml.param.shared import *
 from pyspark.mllib.common import inherit_doc
-from pyspark.mllib.linalg import _convert_to_vector
 
 __all__ = ['KMeans', 'KMeansModel']
 
@@ -35,7 +34,7 @@ class KMeansModel(JavaModel):
 
 
 @inherit_doc
-class KMeans(JavaEstimator, HasFeaturesCol, HasMaxIter, HasSeed):
+class KMeans(JavaEstimator, HasFeaturesCol, HasPredictionCol, HasMaxIter, HasTol, HasSeed):
     """
     K-means clustering with support for multiple parallel runs and a k-means++ like initialization
     mode (the k-means|| algorithm by Bahmani et al). When multiple concurrent runs are requested,
@@ -45,7 +44,7 @@ class KMeans(JavaEstimator, HasFeaturesCol, HasMaxIter, HasSeed):
     >>> data = [(Vectors.dense([0.0, 0.0]),), (Vectors.dense([1.0, 1.0]),),
     ...         (Vectors.dense([9.0, 8.0]),), (Vectors.dense([8.0, 9.0]),)]
     >>> df = sqlContext.createDataFrame(data, ["features"])
-    >>> kmeans = KMeans().setK(2).setSeed(1).setFeaturesCol("features")
+    >>> kmeans = KMeans(k=2, seed=1)
     >>> model = kmeans.fit(df)
     >>> centers = model.clusterCenters()
     >>> len(centers)
@@ -60,10 +59,6 @@ class KMeans(JavaEstimator, HasFeaturesCol, HasMaxIter, HasSeed):
 
     # a placeholder to make it appear in the generated doc
     k = Param(Params._dummy(), "k", "number of clusters to create")
-    epsilon = Param(Params._dummy(), "epsilon",
-                    "distance threshold within which " +
-                    "we've consider centers to have converged")
-    runs = Param(Params._dummy(), "runs", "number of runs of the algorithm to execute in parallel")
     initMode = Param(Params._dummy(), "initMode",
                      "the initialization algorithm. This can be either \"random\" to " +
                      "choose random points as initial cluster centers, or \"k-means||\" " +
@@ -71,21 +66,21 @@ class KMeans(JavaEstimator, HasFeaturesCol, HasMaxIter, HasSeed):
     initSteps = Param(Params._dummy(), "initSteps", "steps for k-means initialization mode")
 
     @keyword_only
-    def __init__(self, k=2, maxIter=20, runs=1, epsilon=1e-4, initMode="k-means||", initStep=5):
+    def __init__(self, featuresCol="features", predictionCol="prediction", k=2,
+                 initMode="k-means||", initSteps=5, tol=1e-4, maxIter=20, seed=None):
+        """
+        __init__(self, featuresCol="features", predictionCol="prediction", k=2, \
+                 initMode="k-means||", initSteps=5, tol=1e-4, maxIter=20, seed=None)
+        """
         super(KMeans, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.clustering.KMeans", self.uid)
         self.k = Param(self, "k", "number of clusters to create")
-        self.epsilon = Param(self, "epsilon",
-                             "distance threshold within which " +
-                             "we've consider centers to have converged")
-        self.runs = Param(self, "runs", "number of runs of the algorithm to execute in parallel")
-        self.seed = Param(self, "seed", "random seed")
         self.initMode = Param(self, "initMode",
                               "the initialization algorithm. This can be either \"random\" to " +
                               "choose random points as initial cluster centers, or \"k-means||\" " +
                               "to use a parallel variant of k-means++")
         self.initSteps = Param(self, "initSteps", "steps for k-means initialization mode")
-        self._setDefault(k=2, maxIter=20, runs=1, epsilon=1e-4, initMode="k-means||", initSteps=5)
+        self._setDefault(k=2, initMode="k-means||", initSteps=5, tol=1e-4, maxIter=20)
         kwargs = self.__init__._input_kwargs
         self.setParams(**kwargs)
 
@@ -93,9 +88,11 @@ class KMeans(JavaEstimator, HasFeaturesCol, HasMaxIter, HasSeed):
         return KMeansModel(java_model)
 
     @keyword_only
-    def setParams(self, k=2, maxIter=20, runs=1, epsilon=1e-4, initMode="k-means||", initSteps=5):
+    def setParams(self, featuresCol="features", predictionCol="prediction", k=2,
+                  initMode="k-means||", initSteps=5, tol=1e-4, maxIter=20, seed=None):
         """
-        setParams(self, k=2, maxIter=20, runs=1, epsilon=1e-4, initMode="k-means||", initSteps=5):
+        setParams(self, featuresCol="features", predictionCol="prediction", k=2, \
+                  initMode="k-means||", initSteps=5, tol=1e-4, maxIter=20, seed=None)
 
         Sets params for KMeans.
         """
@@ -119,40 +116,6 @@ class KMeans(JavaEstimator, HasFeaturesCol, HasMaxIter, HasSeed):
         """
         return self.getOrDefault(self.k)
 
-    def setEpsilon(self, value):
-        """
-        Sets the value of :py:attr:`epsilon`.
-
-        >>> algo = KMeans().setEpsilon(1e-5)
-        >>> abs(algo.getEpsilon() - 1e-5) < 1e-5
-        True
-        """
-        self._paramMap[self.epsilon] = value
-        return self
-
-    def getEpsilon(self):
-        """
-        Gets the value of `epsilon`
-        """
-        return self.getOrDefault(self.epsilon)
-
-    def setRuns(self, value):
-        """
-        Sets the value of :py:attr:`runs`.
-
-        >>> algo = KMeans().setRuns(10)
-        >>> algo.getRuns()
-        10
-        """
-        self._paramMap[self.runs] = value
-        return self
-
-    def getRuns(self):
-        """
-        Gets the value of `runs`
-        """
-        return self.getOrDefault(self.runs)
-
     def setInitMode(self, value):
         """
         Sets the value of :py:attr:`initMode`.