From 8ce645d4eeda203cf5e100c4bdba2d71edd44e6a Mon Sep 17 00:00:00 2001 From: Reynold Xin Date: Tue, 5 Jan 2016 11:10:14 -0800 Subject: [SPARK-12615] Remove some deprecated APIs in RDD/SparkContext I looked at each case individually and it looks like they can all be removed. The only one that I had to think twice was toArray (I even thought about un-deprecating it, until I realized it was a problem in Java to have toArray returning java.util.List). Author: Reynold Xin Closes #10569 from rxin/SPARK-12615. --- .../mllib/JavaBinaryClassificationMetricsExample.java | 12 ++++++------ .../main/scala/org/apache/spark/examples/SparkHdfsLR.scala | 7 +------ .../scala/org/apache/spark/examples/SparkTachyonHdfsLR.scala | 6 +----- 3 files changed, 8 insertions(+), 17 deletions(-) (limited to 'examples') diff --git a/examples/src/main/java/org/apache/spark/examples/mllib/JavaBinaryClassificationMetricsExample.java b/examples/src/main/java/org/apache/spark/examples/mllib/JavaBinaryClassificationMetricsExample.java index 980a9108af..779fac01c4 100644 --- a/examples/src/main/java/org/apache/spark/examples/mllib/JavaBinaryClassificationMetricsExample.java +++ b/examples/src/main/java/org/apache/spark/examples/mllib/JavaBinaryClassificationMetricsExample.java @@ -68,22 +68,22 @@ public class JavaBinaryClassificationMetricsExample { // Precision by threshold JavaRDD> precision = metrics.precisionByThreshold().toJavaRDD(); - System.out.println("Precision by threshold: " + precision.toArray()); + System.out.println("Precision by threshold: " + precision.collect()); // Recall by threshold JavaRDD> recall = metrics.recallByThreshold().toJavaRDD(); - System.out.println("Recall by threshold: " + recall.toArray()); + System.out.println("Recall by threshold: " + recall.collect()); // F Score by threshold JavaRDD> f1Score = metrics.fMeasureByThreshold().toJavaRDD(); - System.out.println("F1 Score by threshold: " + f1Score.toArray()); + System.out.println("F1 Score by threshold: " + f1Score.collect()); JavaRDD> f2Score = metrics.fMeasureByThreshold(2.0).toJavaRDD(); - System.out.println("F2 Score by threshold: " + f2Score.toArray()); + System.out.println("F2 Score by threshold: " + f2Score.collect()); // Precision-recall curve JavaRDD> prc = metrics.pr().toJavaRDD(); - System.out.println("Precision-recall curve: " + prc.toArray()); + System.out.println("Precision-recall curve: " + prc.collect()); // Thresholds JavaRDD thresholds = precision.map( @@ -96,7 +96,7 @@ public class JavaBinaryClassificationMetricsExample { // ROC Curve JavaRDD> roc = metrics.roc().toJavaRDD(); - System.out.println("ROC curve: " + roc.toArray()); + System.out.println("ROC curve: " + roc.collect()); // AUPRC System.out.println("Area under precision-recall curve = " + metrics.areaUnderPR()); diff --git a/examples/src/main/scala/org/apache/spark/examples/SparkHdfsLR.scala b/examples/src/main/scala/org/apache/spark/examples/SparkHdfsLR.scala index 6c90dbec3d..04dec57b71 100644 --- a/examples/src/main/scala/org/apache/spark/examples/SparkHdfsLR.scala +++ b/examples/src/main/scala/org/apache/spark/examples/SparkHdfsLR.scala @@ -26,8 +26,6 @@ import breeze.linalg.{DenseVector, Vector} import org.apache.hadoop.conf.Configuration import org.apache.spark._ -import org.apache.spark.scheduler.InputFormatInfo - /** * Logistic regression based classification. @@ -74,10 +72,7 @@ object SparkHdfsLR { val sparkConf = new SparkConf().setAppName("SparkHdfsLR") val inputPath = args(0) val conf = new Configuration() - val sc = new SparkContext(sparkConf, - InputFormatInfo.computePreferredLocations( - Seq(new InputFormatInfo(conf, classOf[org.apache.hadoop.mapred.TextInputFormat], inputPath)) - )) + val sc = new SparkContext(sparkConf) val lines = sc.textFile(inputPath) val points = lines.map(parsePoint _).cache() val ITERATIONS = args(1).toInt diff --git a/examples/src/main/scala/org/apache/spark/examples/SparkTachyonHdfsLR.scala b/examples/src/main/scala/org/apache/spark/examples/SparkTachyonHdfsLR.scala index e492582710..ddc99d3f90 100644 --- a/examples/src/main/scala/org/apache/spark/examples/SparkTachyonHdfsLR.scala +++ b/examples/src/main/scala/org/apache/spark/examples/SparkTachyonHdfsLR.scala @@ -26,7 +26,6 @@ import breeze.linalg.{DenseVector, Vector} import org.apache.hadoop.conf.Configuration import org.apache.spark._ -import org.apache.spark.scheduler.InputFormatInfo import org.apache.spark.storage.StorageLevel /** @@ -70,10 +69,7 @@ object SparkTachyonHdfsLR { val inputPath = args(0) val sparkConf = new SparkConf().setAppName("SparkTachyonHdfsLR") val conf = new Configuration() - val sc = new SparkContext(sparkConf, - InputFormatInfo.computePreferredLocations( - Seq(new InputFormatInfo(conf, classOf[org.apache.hadoop.mapred.TextInputFormat], inputPath)) - )) + val sc = new SparkContext(sparkConf) val lines = sc.textFile(inputPath) val points = lines.map(parsePoint _).persist(StorageLevel.OFF_HEAP) val ITERATIONS = args(1).toInt -- cgit v1.2.3