diff options
author | Matei Zaharia <matei@eecs.berkeley.edu> | 2013-09-01 00:32:28 -0700 |
---|---|---|
committer | Matei Zaharia <matei@eecs.berkeley.edu> | 2013-09-01 14:13:16 -0700 |
commit | 0a8cc309211c62f8824d76618705c817edcf2424 (patch) | |
tree | 86cb7e3b2611b9287335699dcd535f145ccf56d6 /mllib | |
parent | 5b4dea21439e86b61447bdb1613b2ddff9ffba9f (diff) | |
download | spark-0a8cc309211c62f8824d76618705c817edcf2424.tar.gz spark-0a8cc309211c62f8824d76618705c817edcf2424.tar.bz2 spark-0a8cc309211c62f8824d76618705c817edcf2424.zip |
Move some classes to more appropriate packages:
* RDD, *RDDFunctions -> org.apache.spark.rdd
* Utils, ClosureCleaner, SizeEstimator -> org.apache.spark.util
* JavaSerializer, KryoSerializer -> org.apache.spark.serializer
Diffstat (limited to 'mllib')
21 files changed, 43 insertions, 26 deletions
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/classification/ClassificationModel.scala b/mllib/src/main/scala/org/apache/spark/mllib/classification/ClassificationModel.scala index 4f4a7f5296..60cb44ce89 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/classification/ClassificationModel.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/classification/ClassificationModel.scala @@ -1,6 +1,6 @@ package org.apache.spark.mllib.classification -import org.apache.spark.RDD +import org.apache.spark.rdd.RDD trait ClassificationModel extends Serializable { /** diff --git a/mllib/src/main/scala/org/apache/spark/mllib/classification/LogisticRegression.scala b/mllib/src/main/scala/org/apache/spark/mllib/classification/LogisticRegression.scala index 91bb50c829..50aede9c07 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/classification/LogisticRegression.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/classification/LogisticRegression.scala @@ -19,7 +19,8 @@ package org.apache.spark.mllib.classification import scala.math.round -import org.apache.spark.{Logging, RDD, SparkContext} +import org.apache.spark.SparkContext +import org.apache.spark.rdd.RDD import org.apache.spark.mllib.optimization._ import org.apache.spark.mllib.regression._ import org.apache.spark.mllib.util.MLUtils diff --git a/mllib/src/main/scala/org/apache/spark/mllib/classification/SVM.scala b/mllib/src/main/scala/org/apache/spark/mllib/classification/SVM.scala index c92c7cc3f3..3511e24bce 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/classification/SVM.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/classification/SVM.scala @@ -19,7 +19,8 @@ package org.apache.spark.mllib.classification import scala.math.signum -import org.apache.spark.{Logging, RDD, SparkContext} +import org.apache.spark.SparkContext +import org.apache.spark.rdd.RDD import org.apache.spark.mllib.optimization._ import org.apache.spark.mllib.regression._ import org.apache.spark.mllib.util.MLUtils diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala index 2c3db099fa..edbf77dbcc 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala @@ -20,8 +20,9 @@ package org.apache.spark.mllib.clustering import scala.collection.mutable.ArrayBuffer import scala.util.Random -import org.apache.spark.{SparkContext, RDD} +import org.apache.spark.SparkContext import org.apache.spark.SparkContext._ +import org.apache.spark.rdd.RDD import org.apache.spark.Logging import org.apache.spark.mllib.util.MLUtils diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeansModel.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeansModel.scala index d1fe5d138d..cfc81c985a 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeansModel.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeansModel.scala @@ -17,7 +17,7 @@ package org.apache.spark.mllib.clustering -import org.apache.spark.RDD +import org.apache.spark.rdd.RDD import org.apache.spark.SparkContext._ import org.apache.spark.mllib.util.MLUtils diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala index b62c9b3340..b77364e08d 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala @@ -17,7 +17,8 @@ package org.apache.spark.mllib.optimization -import org.apache.spark.{Logging, RDD, SparkContext} +import org.apache.spark.{Logging, SparkContext} +import org.apache.spark.rdd.RDD import org.apache.spark.SparkContext._ import org.jblas.DoubleMatrix @@ -29,8 +30,9 @@ import scala.collection.mutable.ArrayBuffer * @param gradient Gradient function to be used. * @param updater Updater to be used to update weights after every iteration. */ -class GradientDescent(var gradient: Gradient, var updater: Updater) extends Optimizer { - +class GradientDescent(var gradient: Gradient, var updater: Updater) + extends Optimizer with Logging +{ private var stepSize: Double = 1.0 private var numIterations: Int = 100 private var regParam: Double = 0.0 diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/Optimizer.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/Optimizer.scala index 50059d385d..94d30b56f2 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/Optimizer.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/Optimizer.scala @@ -17,7 +17,7 @@ package org.apache.spark.mllib.optimization -import org.apache.spark.RDD +import org.apache.spark.rdd.RDD trait Optimizer { diff --git a/mllib/src/main/scala/org/apache/spark/mllib/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/mllib/recommendation/ALS.scala index 218217acfe..be002d02bc 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/recommendation/ALS.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/recommendation/ALS.scala @@ -21,9 +21,10 @@ import scala.collection.mutable.{ArrayBuffer, BitSet} import scala.util.Random import scala.util.Sorting -import org.apache.spark.{HashPartitioner, Partitioner, SparkContext, RDD} +import org.apache.spark.{HashPartitioner, Partitioner, SparkContext} import org.apache.spark.storage.StorageLevel -import org.apache.spark.KryoRegistrator +import org.apache.spark.rdd.RDD +import org.apache.spark.serializer.KryoRegistrator import org.apache.spark.SparkContext._ import com.esotericsoftware.kryo.Kryo @@ -432,7 +433,7 @@ object ALS { val (master, ratingsFile, rank, iters, outputDir) = (args(0), args(1), args(2).toInt, args(3).toInt, args(4)) val blocks = if (args.length == 6) args(5).toInt else -1 - System.setProperty("spark.serializer", "org.apache.spark.KryoSerializer") + System.setProperty("spark.serializer", "org.apache.spark.serializer.KryoSerializer") System.setProperty("spark.kryo.registrator", classOf[ALSRegistrator].getName) System.setProperty("spark.kryo.referenceTracking", "false") System.setProperty("spark.kryoserializer.buffer.mb", "8") diff --git a/mllib/src/main/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModel.scala b/mllib/src/main/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModel.scala index ae9fe48aec..af43d89c70 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModel.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModel.scala @@ -17,7 +17,7 @@ package org.apache.spark.mllib.recommendation -import org.apache.spark.RDD +import org.apache.spark.rdd.RDD import org.apache.spark.SparkContext._ import org.jblas._ diff --git a/mllib/src/main/scala/org/apache/spark/mllib/regression/GeneralizedLinearAlgorithm.scala b/mllib/src/main/scala/org/apache/spark/mllib/regression/GeneralizedLinearAlgorithm.scala index 06015110ac..f98b0b536d 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/regression/GeneralizedLinearAlgorithm.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/regression/GeneralizedLinearAlgorithm.scala @@ -17,7 +17,8 @@ package org.apache.spark.mllib.regression -import org.apache.spark.{Logging, RDD, SparkException} +import org.apache.spark.{Logging, SparkException} +import org.apache.spark.rdd.RDD import org.apache.spark.mllib.optimization._ import org.jblas.DoubleMatrix diff --git a/mllib/src/main/scala/org/apache/spark/mllib/regression/Lasso.scala b/mllib/src/main/scala/org/apache/spark/mllib/regression/Lasso.scala index df3beb1959..d959695325 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/regression/Lasso.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/regression/Lasso.scala @@ -17,7 +17,8 @@ package org.apache.spark.mllib.regression -import org.apache.spark.{Logging, RDD, SparkContext} +import org.apache.spark.{Logging, SparkContext} +import org.apache.spark.rdd.RDD import org.apache.spark.mllib.optimization._ import org.apache.spark.mllib.util.MLUtils diff --git a/mllib/src/main/scala/org/apache/spark/mllib/regression/LinearRegression.scala b/mllib/src/main/scala/org/apache/spark/mllib/regression/LinearRegression.scala index 71f968471c..ae95ea24fc 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/regression/LinearRegression.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/regression/LinearRegression.scala @@ -17,7 +17,8 @@ package org.apache.spark.mllib.regression -import org.apache.spark.{Logging, RDD, SparkContext} +import org.apache.spark.{Logging, SparkContext} +import org.apache.spark.rdd.RDD import org.apache.spark.mllib.optimization._ import org.apache.spark.mllib.util.MLUtils diff --git a/mllib/src/main/scala/org/apache/spark/mllib/regression/RegressionModel.scala b/mllib/src/main/scala/org/apache/spark/mllib/regression/RegressionModel.scala index 8dd325efc0..423afc32d6 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/regression/RegressionModel.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/regression/RegressionModel.scala @@ -17,7 +17,7 @@ package org.apache.spark.mllib.regression -import org.apache.spark.RDD +import org.apache.spark.rdd.RDD trait RegressionModel extends Serializable { /** diff --git a/mllib/src/main/scala/org/apache/spark/mllib/regression/RidgeRegression.scala b/mllib/src/main/scala/org/apache/spark/mllib/regression/RidgeRegression.scala index 228ab9e4e8..b29508d2b9 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/regression/RidgeRegression.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/regression/RidgeRegression.scala @@ -17,7 +17,8 @@ package org.apache.spark.mllib.regression -import org.apache.spark.{Logging, RDD, SparkContext} +import org.apache.spark.{Logging, SparkContext} +import org.apache.spark.rdd.RDD import org.apache.spark.mllib.optimization._ import org.apache.spark.mllib.util.MLUtils diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/DataValidators.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/DataValidators.scala index 7fd4623071..8b55bce7c4 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/util/DataValidators.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/util/DataValidators.scala @@ -17,7 +17,8 @@ package org.apache.spark.mllib.util -import org.apache.spark.{RDD, Logging} +import org.apache.spark.Logging +import org.apache.spark.rdd.RDD import org.apache.spark.mllib.regression.LabeledPoint /** diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/KMeansDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/KMeansDataGenerator.scala index 6500d47183..9109189dff 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/util/KMeansDataGenerator.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/util/KMeansDataGenerator.scala @@ -19,7 +19,8 @@ package org.apache.spark.mllib.util import scala.util.Random -import org.apache.spark.{RDD, SparkContext} +import org.apache.spark.SparkContext +import org.apache.spark.rdd.RDD /** * Generate test data for KMeans. This class first chooses k cluster centers diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala index 4c49d484b4..bc5045fb05 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala @@ -22,7 +22,8 @@ import scala.util.Random import org.jblas.DoubleMatrix -import org.apache.spark.{RDD, SparkContext} +import org.apache.spark.SparkContext +import org.apache.spark.rdd.RDD import org.apache.spark.mllib.regression.LabeledPoint import org.apache.spark.mllib.regression.LabeledPoint diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/LogisticRegressionDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/LogisticRegressionDataGenerator.scala index f553298fc5..52c4a71d62 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/util/LogisticRegressionDataGenerator.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/util/LogisticRegressionDataGenerator.scala @@ -19,7 +19,8 @@ package org.apache.spark.mllib.util import scala.util.Random -import org.apache.spark.{RDD, SparkContext} +import org.apache.spark.SparkContext +import org.apache.spark.rdd.RDD import org.apache.spark.mllib.regression.LabeledPoint /** diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala index 7eb69ae81c..5aec867257 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala @@ -21,7 +21,8 @@ import scala.util.Random import org.jblas.DoubleMatrix -import org.apache.spark.{RDD, SparkContext} +import org.apache.spark.SparkContext +import org.apache.spark.rdd.RDD import org.apache.spark.mllib.util.MLUtils /** diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala index 0aeafbe23c..d91b74c3ac 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala @@ -17,7 +17,8 @@ package org.apache.spark.mllib.util -import org.apache.spark.{RDD, SparkContext} +import org.apache.spark.SparkContext +import org.apache.spark.rdd.RDD import org.apache.spark.SparkContext._ import org.jblas.DoubleMatrix diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/SVMDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/SVMDataGenerator.scala index d3f191b05b..6e9f667635 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/util/SVMDataGenerator.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/util/SVMDataGenerator.scala @@ -2,9 +2,10 @@ package org.apache.spark.mllib.util import scala.util.Random -import org.apache.spark.{RDD, SparkContext} - import org.jblas.DoubleMatrix + +import org.apache.spark.SparkContext +import org.apache.spark.rdd.RDD import org.apache.spark.mllib.regression.LabeledPoint /** |