aboutsummaryrefslogtreecommitdiff
path: root/examples/src/main/java
diff options
context:
space:
mode:
authorDongjoon Hyun <dongjoon@apache.org>2016-05-04 14:31:36 -0700
committerAndrew Or <andrew@databricks.com>2016-05-04 14:31:36 -0700
commitcdce4e62a5674e2034e5d395578b1a60e3d8c435 (patch)
treec715f2555dad353683f82820962576f89b2db452 /examples/src/main/java
parentcf2e9da612397233ae7bca0e9ce57309f16226b5 (diff)
downloadspark-cdce4e62a5674e2034e5d395578b1a60e3d8c435.tar.gz
spark-cdce4e62a5674e2034e5d395578b1a60e3d8c435.tar.bz2
spark-cdce4e62a5674e2034e5d395578b1a60e3d8c435.zip
[SPARK-15031][EXAMPLE] Use SparkSession in Scala/Python/Java example.
## What changes were proposed in this pull request? This PR aims to update Scala/Python/Java examples by replacing `SQLContext` with newly added `SparkSession`. - Use **SparkSession Builder Pattern** in 154(Scala 55, Java 52, Python 47) files. - Add `getConf` in Python SparkContext class: `python/pyspark/context.py` - Replace **SQLContext Singleton Pattern** with **SparkSession Singleton Pattern**: - `SqlNetworkWordCount.scala` - `JavaSqlNetworkWordCount.java` - `sql_network_wordcount.py` Now, `SQLContexts` are used only in R examples and the following two Python examples. The python examples are untouched in this PR since it already fails some unknown issue. - `simple_params_example.py` - `aft_survival_regression.py` ## How was this patch tested? Manual. Author: Dongjoon Hyun <dongjoon@apache.org> Closes #12809 from dongjoon-hyun/SPARK-15031.
Diffstat (limited to 'examples/src/main/java')
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaAFTSurvivalRegressionExample.java12
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaALSExample.java15
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaBinarizerExample.java15
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaBisectingKMeansExample.java18
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaBucketizerExample.java18
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaChiSqSelectorExample.java15
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaCountVectorizerExample.java19
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaDCTExample.java15
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaDecisionTreeClassificationExample.java13
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaDecisionTreeRegressionExample.java13
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaDeveloperApiExample.java15
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaElementwiseProductExample.java15
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaEstimatorTransformerParamExample.java16
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaGradientBoostedTreeClassifierExample.java11
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaGradientBoostedTreeRegressorExample.java14
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaIndexToStringExample.java18
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaKMeansExample.java14
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaLDAExample.java14
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaLinearRegressionWithElasticNetExample.java13
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaLogisticRegressionSummaryExample.java13
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaLogisticRegressionWithElasticNetExample.java13
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaMaxAbsScalerExample.java12
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaMinMaxScalerExample.java12
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaModelSelectionViaCrossValidationExample.java16
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaModelSelectionViaTrainValidationSplitExample.java14
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaMultilayerPerceptronClassifierExample.java13
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaNGramExample.java18
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaNaiveBayesExample.java14
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaNormalizerExample.java13
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaOneHotEncoderExample.java18
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaOneVsRestExample.java14
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaPCAExample.java18
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaPipelineExample.java16
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaPolynomialExpansionExample.java17
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaQuantileDiscretizerExample.java29
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaRFormulaExample.java18
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaRandomForestClassifierExample.java14
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaRandomForestRegressorExample.java14
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaSQLTransformerExample.java19
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaSimpleParamsExample.java14
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaSimpleTextClassificationPipeline.java15
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaStandardScalerExample.java13
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaStopWordsRemoverExample.java18
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaStringIndexerExample.java18
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaTfIdfExample.java18
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaTokenizerExample.java18
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaVectorAssemblerExample.java14
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaVectorIndexerExample.java12
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaVectorSlicerExample.java19
-rw-r--r--examples/src/main/java/org/apache/spark/examples/ml/JavaWord2VecExample.java19
-rw-r--r--examples/src/main/java/org/apache/spark/examples/sql/JavaSparkSQL.java33
-rw-r--r--examples/src/main/java/org/apache/spark/examples/streaming/JavaSqlNetworkWordCount.java19
52 files changed, 319 insertions, 509 deletions
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaAFTSurvivalRegressionExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaAFTSurvivalRegressionExample.java
index 22b93a3a85..ecb7084e03 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaAFTSurvivalRegressionExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaAFTSurvivalRegressionExample.java
@@ -21,23 +21,19 @@ package org.apache.spark.examples.ml;
import java.util.Arrays;
import java.util.List;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.ml.regression.AFTSurvivalRegression;
import org.apache.spark.ml.regression.AFTSurvivalRegressionModel;
import org.apache.spark.mllib.linalg.*;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.types.*;
// $example off$
public class JavaAFTSurvivalRegressionExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaAFTSurvivalRegressionExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext jsql = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaAFTSurvivalRegressionExample").getOrCreate();
// $example on$
List<Row> data = Arrays.asList(
@@ -52,7 +48,7 @@ public class JavaAFTSurvivalRegressionExample {
new StructField("censor", DataTypes.DoubleType, false, Metadata.empty()),
new StructField("features", new VectorUDT(), false, Metadata.empty())
});
- Dataset<Row> training = jsql.createDataFrame(data, schema);
+ Dataset<Row> training = spark.createDataFrame(data, schema);
double[] quantileProbabilities = new double[]{0.3, 0.6};
AFTSurvivalRegression aft = new AFTSurvivalRegression()
.setQuantileProbabilities(quantileProbabilities)
@@ -66,6 +62,6 @@ public class JavaAFTSurvivalRegressionExample {
model.transform(training).show(false);
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaALSExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaALSExample.java
index 088037d427..9a9a10489b 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaALSExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaALSExample.java
@@ -17,11 +17,9 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example on$
import java.io.Serializable;
@@ -83,18 +81,17 @@ public class JavaALSExample {
// $example off$
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaALSExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaALSExample").getOrCreate();
// $example on$
- JavaRDD<Rating> ratingsRDD = jsc.textFile("data/mllib/als/sample_movielens_ratings.txt")
+ JavaRDD<Rating> ratingsRDD = spark
+ .read().text("data/mllib/als/sample_movielens_ratings.txt").javaRDD()
.map(new Function<String, Rating>() {
public Rating call(String str) {
return Rating.parseRating(str);
}
});
- Dataset<Row> ratings = sqlContext.createDataFrame(ratingsRDD, Rating.class);
+ Dataset<Row> ratings = spark.createDataFrame(ratingsRDD, Rating.class);
Dataset<Row>[] splits = ratings.randomSplit(new double[]{0.8, 0.2});
Dataset<Row> training = splits[0];
Dataset<Row> test = splits[1];
@@ -121,6 +118,6 @@ public class JavaALSExample {
Double rmse = evaluator.evaluate(predictions);
System.out.println("Root-mean-square error = " + rmse);
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaBinarizerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaBinarizerExample.java
index 0a6e9c2a1f..88e4298a61 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaBinarizerExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaBinarizerExample.java
@@ -20,10 +20,11 @@ package org.apache.spark.examples.ml;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example on$
import java.util.Arrays;
+import java.util.List;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.ml.feature.Binarizer;
@@ -37,21 +38,19 @@ import org.apache.spark.sql.types.StructType;
public class JavaBinarizerExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaBinarizerExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext jsql = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaBinarizerExample").getOrCreate();
// $example on$
- JavaRDD<Row> jrdd = jsc.parallelize(Arrays.asList(
+ List<Row> data = Arrays.asList(
RowFactory.create(0, 0.1),
RowFactory.create(1, 0.8),
RowFactory.create(2, 0.2)
- ));
+ );
StructType schema = new StructType(new StructField[]{
new StructField("label", DataTypes.DoubleType, false, Metadata.empty()),
new StructField("feature", DataTypes.DoubleType, false, Metadata.empty())
});
- Dataset<Row> continuousDataFrame = jsql.createDataFrame(jrdd, schema);
+ Dataset<Row> continuousDataFrame = spark.createDataFrame(data, schema);
Binarizer binarizer = new Binarizer()
.setInputCol("feature")
.setOutputCol("binarized_feature")
@@ -63,6 +62,6 @@ public class JavaBinarizerExample {
System.out.println(binarized_value);
}
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaBisectingKMeansExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaBisectingKMeansExample.java
index 1d1a518bbc..51aa35084e 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaBisectingKMeansExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaBisectingKMeansExample.java
@@ -18,12 +18,10 @@
package org.apache.spark.examples.ml;
import java.util.Arrays;
+import java.util.List;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaRDD;
-import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.RowFactory;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example on$
import org.apache.spark.ml.clustering.BisectingKMeans;
import org.apache.spark.ml.clustering.BisectingKMeansModel;
@@ -44,25 +42,23 @@ import org.apache.spark.sql.types.StructType;
public class JavaBisectingKMeansExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaBisectingKMeansExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext jsql = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaBisectingKMeansExample").getOrCreate();
// $example on$
- JavaRDD<Row> data = jsc.parallelize(Arrays.asList(
+ List<Row> data = Arrays.asList(
RowFactory.create(Vectors.dense(0.1, 0.1, 0.1)),
RowFactory.create(Vectors.dense(0.3, 0.3, 0.25)),
RowFactory.create(Vectors.dense(0.1, 0.1, -0.1)),
RowFactory.create(Vectors.dense(20.3, 20.1, 19.9)),
RowFactory.create(Vectors.dense(20.2, 20.1, 19.7)),
RowFactory.create(Vectors.dense(18.9, 20.0, 19.7))
- ));
+ );
StructType schema = new StructType(new StructField[]{
new StructField("features", new VectorUDT(), false, Metadata.empty()),
});
- Dataset<Row> dataset = jsql.createDataFrame(data, schema);
+ Dataset<Row> dataset = spark.createDataFrame(data, schema);
BisectingKMeans bkm = new BisectingKMeans().setK(2);
BisectingKMeansModel model = bkm.fit(dataset);
@@ -76,6 +72,6 @@ public class JavaBisectingKMeansExample {
}
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaBucketizerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaBucketizerExample.java
index 68ffa702ea..0c24f52cf5 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaBucketizerExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaBucketizerExample.java
@@ -17,14 +17,12 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example on$
import java.util.Arrays;
+import java.util.List;
-import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.ml.feature.Bucketizer;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
@@ -37,23 +35,21 @@ import org.apache.spark.sql.types.StructType;
public class JavaBucketizerExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaBucketizerExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext jsql = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaBucketizerExample").getOrCreate();
// $example on$
double[] splits = {Double.NEGATIVE_INFINITY, -0.5, 0.0, 0.5, Double.POSITIVE_INFINITY};
- JavaRDD<Row> data = jsc.parallelize(Arrays.asList(
+ List<Row> data = Arrays.asList(
RowFactory.create(-0.5),
RowFactory.create(-0.3),
RowFactory.create(0.0),
RowFactory.create(0.2)
- ));
+ );
StructType schema = new StructType(new StructField[]{
new StructField("features", DataTypes.DoubleType, false, Metadata.empty())
});
- Dataset<Row> dataFrame = jsql.createDataFrame(data, schema);
+ Dataset<Row> dataFrame = spark.createDataFrame(data, schema);
Bucketizer bucketizer = new Bucketizer()
.setInputCol("features")
@@ -64,7 +60,7 @@ public class JavaBucketizerExample {
Dataset<Row> bucketedData = bucketizer.transform(dataFrame);
bucketedData.show();
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaChiSqSelectorExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaChiSqSelectorExample.java
index b1bf1cfeb2..684cf9a714 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaChiSqSelectorExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaChiSqSelectorExample.java
@@ -21,10 +21,11 @@ import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example on$
import java.util.Arrays;
+import java.util.List;
import org.apache.spark.ml.feature.ChiSqSelector;
import org.apache.spark.mllib.linalg.VectorUDT;
@@ -39,23 +40,21 @@ import org.apache.spark.sql.types.StructType;
public class JavaChiSqSelectorExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaChiSqSelectorExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaChiSqSelectorExample").getOrCreate();
// $example on$
- JavaRDD<Row> jrdd = jsc.parallelize(Arrays.asList(
+ List<Row> data = Arrays.asList(
RowFactory.create(7, Vectors.dense(0.0, 0.0, 18.0, 1.0), 1.0),
RowFactory.create(8, Vectors.dense(0.0, 1.0, 12.0, 0.0), 0.0),
RowFactory.create(9, Vectors.dense(1.0, 0.0, 15.0, 0.1), 0.0)
- ));
+ );
StructType schema = new StructType(new StructField[]{
new StructField("id", DataTypes.IntegerType, false, Metadata.empty()),
new StructField("features", new VectorUDT(), false, Metadata.empty()),
new StructField("clicked", DataTypes.DoubleType, false, Metadata.empty())
});
- Dataset<Row> df = sqlContext.createDataFrame(jrdd, schema);
+ Dataset<Row> df = spark.createDataFrame(data, schema);
ChiSqSelector selector = new ChiSqSelector()
.setNumTopFeatures(1)
@@ -66,6 +65,6 @@ public class JavaChiSqSelectorExample {
Dataset<Row> result = selector.fit(df).transform(df);
result.show();
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaCountVectorizerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaCountVectorizerExample.java
index ec3ac202be..0631f9d6d5 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaCountVectorizerExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaCountVectorizerExample.java
@@ -19,36 +19,31 @@ package org.apache.spark.examples.ml;
// $example on$
import java.util.Arrays;
+import java.util.List;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaRDD;
-import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.ml.feature.CountVectorizer;
import org.apache.spark.ml.feature.CountVectorizerModel;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.types.*;
// $example off$
public class JavaCountVectorizerExample {
public static void main(String[] args) {
-
- SparkConf conf = new SparkConf().setAppName("JavaCountVectorizerExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaCountVectorizerExample").getOrCreate();
// $example on$
// Input data: Each row is a bag of words from a sentence or document.
- JavaRDD<Row> jrdd = jsc.parallelize(Arrays.asList(
+ List<Row> data = Arrays.asList(
RowFactory.create(Arrays.asList("a", "b", "c")),
RowFactory.create(Arrays.asList("a", "b", "b", "c", "a"))
- ));
+ );
StructType schema = new StructType(new StructField [] {
new StructField("text", new ArrayType(DataTypes.StringType, true), false, Metadata.empty())
});
- Dataset<Row> df = sqlContext.createDataFrame(jrdd, schema);
+ Dataset<Row> df = spark.createDataFrame(data, schema);
// fit a CountVectorizerModel from the corpus
CountVectorizerModel cvModel = new CountVectorizer()
@@ -66,6 +61,6 @@ public class JavaCountVectorizerExample {
cvModel.transform(df).show();
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaDCTExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaDCTExample.java
index 4b15fde9c3..ec57a24451 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaDCTExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaDCTExample.java
@@ -20,10 +20,11 @@ package org.apache.spark.examples.ml;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example on$
import java.util.Arrays;
+import java.util.List;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.ml.feature.DCT;
@@ -38,20 +39,18 @@ import org.apache.spark.sql.types.StructType;
public class JavaDCTExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaDCTExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext jsql = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaDCTExample").getOrCreate();
// $example on$
- JavaRDD<Row> data = jsc.parallelize(Arrays.asList(
+ List<Row> data = Arrays.asList(
RowFactory.create(Vectors.dense(0.0, 1.0, -2.0, 3.0)),
RowFactory.create(Vectors.dense(-1.0, 2.0, 4.0, -7.0)),
RowFactory.create(Vectors.dense(14.0, -2.0, -5.0, 1.0))
- ));
+ );
StructType schema = new StructType(new StructField[]{
new StructField("features", new VectorUDT(), false, Metadata.empty()),
});
- Dataset<Row> df = jsql.createDataFrame(data, schema);
+ Dataset<Row> df = spark.createDataFrame(data, schema);
DCT dct = new DCT()
.setInputCol("features")
.setOutputCol("featuresDCT")
@@ -59,7 +58,7 @@ public class JavaDCTExample {
Dataset<Row> dctDf = dct.transform(df);
dctDf.select("featuresDCT").show(3);
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaDecisionTreeClassificationExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaDecisionTreeClassificationExample.java
index 8214952f80..733bc4181c 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaDecisionTreeClassificationExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaDecisionTreeClassificationExample.java
@@ -17,8 +17,6 @@
// scalastyle:off println
package org.apache.spark.examples.ml;
// $example on$
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.ml.Pipeline;
import org.apache.spark.ml.PipelineModel;
import org.apache.spark.ml.PipelineStage;
@@ -28,18 +26,17 @@ import org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator;
import org.apache.spark.ml.feature.*;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example off$
public class JavaDecisionTreeClassificationExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaDecisionTreeClassificationExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession
+ .builder().appName("JavaDecisionTreeClassificationExample").getOrCreate();
// $example on$
// Load the data stored in LIBSVM format as a DataFrame.
- Dataset<Row> data = sqlContext
+ Dataset<Row> data = spark
.read()
.format("libsvm")
.load("data/mllib/sample_libsvm_data.txt");
@@ -100,6 +97,6 @@ public class JavaDecisionTreeClassificationExample {
System.out.println("Learned classification tree model:\n" + treeModel.toDebugString());
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaDecisionTreeRegressionExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaDecisionTreeRegressionExample.java
index a4f3e97bf3..bd6dc3edd3 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaDecisionTreeRegressionExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaDecisionTreeRegressionExample.java
@@ -17,8 +17,6 @@
// scalastyle:off println
package org.apache.spark.examples.ml;
// $example on$
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.ml.Pipeline;
import org.apache.spark.ml.PipelineModel;
import org.apache.spark.ml.PipelineStage;
@@ -29,17 +27,16 @@ import org.apache.spark.ml.regression.DecisionTreeRegressionModel;
import org.apache.spark.ml.regression.DecisionTreeRegressor;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example off$
public class JavaDecisionTreeRegressionExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaDecisionTreeRegressionExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession
+ .builder().appName("JavaDecisionTreeRegressionExample").getOrCreate();
// $example on$
// Load the data stored in LIBSVM format as a DataFrame.
- Dataset<Row> data = sqlContext.read().format("libsvm")
+ Dataset<Row> data = spark.read().format("libsvm")
.load("data/mllib/sample_libsvm_data.txt");
// Automatically identify categorical features, and index them.
@@ -85,6 +82,6 @@ public class JavaDecisionTreeRegressionExample {
System.out.println("Learned regression tree model:\n" + treeModel.toDebugString());
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaDeveloperApiExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaDeveloperApiExample.java
index 0ba94786d4..90023ac06b 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaDeveloperApiExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaDeveloperApiExample.java
@@ -21,9 +21,7 @@ import java.util.List;
import com.google.common.collect.Lists;
-import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
-import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.ml.classification.Classifier;
import org.apache.spark.ml.classification.ClassificationModel;
import org.apache.spark.ml.param.IntParam;
@@ -35,7 +33,7 @@ import org.apache.spark.mllib.linalg.Vectors;
import org.apache.spark.mllib.regression.LabeledPoint;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
/**
@@ -51,9 +49,7 @@ import org.apache.spark.sql.SQLContext;
public class JavaDeveloperApiExample {
public static void main(String[] args) throws Exception {
- SparkConf conf = new SparkConf().setAppName("JavaDeveloperApiExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext jsql = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaDeveloperApiExample").getOrCreate();
// Prepare training data.
List<LabeledPoint> localTraining = Lists.newArrayList(
@@ -61,8 +57,7 @@ public class JavaDeveloperApiExample {
new LabeledPoint(0.0, Vectors.dense(2.0, 1.0, -1.0)),
new LabeledPoint(0.0, Vectors.dense(2.0, 1.3, 1.0)),
new LabeledPoint(1.0, Vectors.dense(0.0, 1.2, -0.5)));
- Dataset<Row> training = jsql.createDataFrame(
- jsc.parallelize(localTraining), LabeledPoint.class);
+ Dataset<Row> training = spark.createDataFrame(localTraining, LabeledPoint.class);
// Create a LogisticRegression instance. This instance is an Estimator.
MyJavaLogisticRegression lr = new MyJavaLogisticRegression();
@@ -80,7 +75,7 @@ public class JavaDeveloperApiExample {
new LabeledPoint(1.0, Vectors.dense(-1.0, 1.5, 1.3)),
new LabeledPoint(0.0, Vectors.dense(3.0, 2.0, -0.1)),
new LabeledPoint(1.0, Vectors.dense(0.0, 2.2, -1.5)));
- Dataset<Row> test = jsql.createDataFrame(jsc.parallelize(localTest), LabeledPoint.class);
+ Dataset<Row> test = spark.createDataFrame(localTest, LabeledPoint.class);
// Make predictions on test documents. cvModel uses the best model found (lrModel).
Dataset<Row> results = model.transform(test);
@@ -93,7 +88,7 @@ public class JavaDeveloperApiExample {
" even though all coefficients are 0!");
}
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaElementwiseProductExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaElementwiseProductExample.java
index 37de9cf359..a062a6fcd0 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaElementwiseProductExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaElementwiseProductExample.java
@@ -20,7 +20,7 @@ package org.apache.spark.examples.ml;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example on$
import java.util.ArrayList;
@@ -41,16 +41,15 @@ import org.apache.spark.sql.types.StructType;
public class JavaElementwiseProductExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaElementwiseProductExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession
+ .builder().appName("JavaElementwiseProductExample").getOrCreate();
// $example on$
// Create some vector data; also works for sparse vectors
- JavaRDD<Row> jrdd = jsc.parallelize(Arrays.asList(
+ List<Row> data = Arrays.asList(
RowFactory.create("a", Vectors.dense(1.0, 2.0, 3.0)),
RowFactory.create("b", Vectors.dense(4.0, 5.0, 6.0))
- ));
+ );
List<StructField> fields = new ArrayList<>(2);
fields.add(DataTypes.createStructField("id", DataTypes.StringType, false));
@@ -58,7 +57,7 @@ public class JavaElementwiseProductExample {
StructType schema = DataTypes.createStructType(fields);
- Dataset<Row> dataFrame = sqlContext.createDataFrame(jrdd, schema);
+ Dataset<Row> dataFrame = spark.createDataFrame(data, schema);
Vector transformingVector = Vectors.dense(0.0, 1.0, 2.0);
@@ -70,6 +69,6 @@ public class JavaElementwiseProductExample {
// Batch transform the vectors to create new column:
transformer.transform(dataFrame).show();
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaEstimatorTransformerParamExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaEstimatorTransformerParamExample.java
index 604b193dd4..5ba8e6cf44 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaEstimatorTransformerParamExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaEstimatorTransformerParamExample.java
@@ -21,8 +21,6 @@ package org.apache.spark.examples.ml;
import java.util.Arrays;
// $example off$
-import org.apache.spark.SparkConf;
-import org.apache.spark.SparkContext;
// $example on$
import org.apache.spark.ml.classification.LogisticRegression;
import org.apache.spark.ml.classification.LogisticRegressionModel;
@@ -32,23 +30,21 @@ import org.apache.spark.mllib.regression.LabeledPoint;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
// $example off$
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
/**
* Java example for Estimator, Transformer, and Param.
*/
public class JavaEstimatorTransformerParamExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf()
- .setAppName("JavaEstimatorTransformerParamExample");
- SparkContext sc = new SparkContext(conf);
- SQLContext sqlContext = new SQLContext(sc);
+ SparkSession spark = SparkSession
+ .builder().appName("JavaEstimatorTransformerParamExample").getOrCreate();
// $example on$
// Prepare training data.
// We use LabeledPoint, which is a JavaBean. Spark SQL can convert RDDs of JavaBeans into
// DataFrames, where it uses the bean metadata to infer the schema.
- Dataset<Row> training = sqlContext.createDataFrame(
+ Dataset<Row> training = spark.createDataFrame(
Arrays.asList(
new LabeledPoint(1.0, Vectors.dense(0.0, 1.1, 0.1)),
new LabeledPoint(0.0, Vectors.dense(2.0, 1.0, -1.0)),
@@ -89,7 +85,7 @@ public class JavaEstimatorTransformerParamExample {
System.out.println("Model 2 was fit using parameters: " + model2.parent().extractParamMap());
// Prepare test documents.
- Dataset<Row> test = sqlContext.createDataFrame(Arrays.asList(
+ Dataset<Row> test = spark.createDataFrame(Arrays.asList(
new LabeledPoint(1.0, Vectors.dense(-1.0, 1.5, 1.3)),
new LabeledPoint(0.0, Vectors.dense(3.0, 2.0, -0.1)),
new LabeledPoint(1.0, Vectors.dense(0.0, 2.2, -1.5))
@@ -107,6 +103,6 @@ public class JavaEstimatorTransformerParamExample {
}
// $example off$
- sc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaGradientBoostedTreeClassifierExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaGradientBoostedTreeClassifierExample.java
index 553070dace..a7c89b9d19 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaGradientBoostedTreeClassifierExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaGradientBoostedTreeClassifierExample.java
@@ -29,18 +29,17 @@ import org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator;
import org.apache.spark.ml.feature.*;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example off$
public class JavaGradientBoostedTreeClassifierExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaGradientBoostedTreeClassifierExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession
+ .builder().appName("JavaGradientBoostedTreeClassifierExample").getOrCreate();
// $example on$
// Load and parse the data file, converting it to a DataFrame.
- Dataset<Row> data = sqlContext.read().format("libsvm")
+ Dataset<Row> data = spark.read().format("libsvm")
.load("data/mllib/sample_libsvm_data.txt");
// Index labels, adding metadata to the label column.
@@ -99,6 +98,6 @@ public class JavaGradientBoostedTreeClassifierExample {
System.out.println("Learned classification GBT model:\n" + gbtModel.toDebugString());
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaGradientBoostedTreeRegressorExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaGradientBoostedTreeRegressorExample.java
index 83fd89e3bd..6d3f21fdaf 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaGradientBoostedTreeRegressorExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaGradientBoostedTreeRegressorExample.java
@@ -17,8 +17,6 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
// $example on$
import org.apache.spark.ml.Pipeline;
import org.apache.spark.ml.PipelineModel;
@@ -30,19 +28,17 @@ import org.apache.spark.ml.regression.GBTRegressionModel;
import org.apache.spark.ml.regression.GBTRegressor;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example off$
public class JavaGradientBoostedTreeRegressorExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaGradientBoostedTreeRegressorExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession
+ .builder().appName("JavaGradientBoostedTreeRegressorExample").getOrCreate();
// $example on$
// Load and parse the data file, converting it to a DataFrame.
- Dataset<Row> data =
- sqlContext.read().format("libsvm").load("data/mllib/sample_libsvm_data.txt");
+ Dataset<Row> data = spark.read().format("libsvm").load("data/mllib/sample_libsvm_data.txt");
// Automatically identify categorical features, and index them.
// Set maxCategories so features with > 4 distinct values are treated as continuous.
@@ -87,6 +83,6 @@ public class JavaGradientBoostedTreeRegressorExample {
System.out.println("Learned regression GBT model:\n" + gbtModel.toDebugString());
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaIndexToStringExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaIndexToStringExample.java
index 9b8c22f3bd..ccd74f2920 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaIndexToStringExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaIndexToStringExample.java
@@ -17,14 +17,12 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaRDD;
-import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example on$
import java.util.Arrays;
+import java.util.List;
import org.apache.spark.ml.feature.IndexToString;
import org.apache.spark.ml.feature.StringIndexer;
@@ -39,24 +37,22 @@ import org.apache.spark.sql.types.StructType;
public class JavaIndexToStringExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaIndexToStringExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaIndexToStringExample").getOrCreate();
// $example on$
- JavaRDD<Row> jrdd = jsc.parallelize(Arrays.asList(
+ List<Row> data = Arrays.asList(
RowFactory.create(0, "a"),
RowFactory.create(1, "b"),
RowFactory.create(2, "c"),
RowFactory.create(3, "a"),
RowFactory.create(4, "a"),
RowFactory.create(5, "c")
- ));
+ );
StructType schema = new StructType(new StructField[]{
new StructField("id", DataTypes.IntegerType, false, Metadata.empty()),
new StructField("category", DataTypes.StringType, false, Metadata.empty())
});
- Dataset<Row> df = sqlContext.createDataFrame(jrdd, schema);
+ Dataset<Row> df = spark.createDataFrame(data, schema);
StringIndexerModel indexer = new StringIndexer()
.setInputCol("category")
@@ -70,6 +66,6 @@ public class JavaIndexToStringExample {
Dataset<Row> converted = converter.transform(indexed);
converted.select("id", "originalCategory").show();
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaKMeansExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaKMeansExample.java
index c5022f4c0b..e6d82a0513 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaKMeansExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaKMeansExample.java
@@ -19,12 +19,10 @@ package org.apache.spark.examples.ml;
import java.util.regex.Pattern;
-import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
-import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.catalyst.expressions.GenericRow;
// $example on$
import org.apache.spark.ml.clustering.KMeansModel;
@@ -72,16 +70,14 @@ public class JavaKMeansExample {
int k = Integer.parseInt(args[1]);
// Parses the arguments
- SparkConf conf = new SparkConf().setAppName("JavaKMeansExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaKMeansExample").getOrCreate();
// $example on$
// Loads data
- JavaRDD<Row> points = jsc.textFile(inputFile).map(new ParsePoint());
+ JavaRDD<Row> points = spark.read().text(inputFile).javaRDD().map(new ParsePoint());
StructField[] fields = {new StructField("features", new VectorUDT(), false, Metadata.empty())};
StructType schema = new StructType(fields);
- Dataset<Row> dataset = sqlContext.createDataFrame(points, schema);
+ Dataset<Row> dataset = spark.createDataFrame(points, schema);
// Trains a k-means model
KMeans kmeans = new KMeans()
@@ -96,6 +92,6 @@ public class JavaKMeansExample {
}
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaLDAExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaLDAExample.java
index 351bc40118..b8baca5920 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaLDAExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaLDAExample.java
@@ -19,9 +19,7 @@ package org.apache.spark.examples.ml;
// $example on$
import java.util.regex.Pattern;
-import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
-import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.ml.clustering.LDA;
import org.apache.spark.ml.clustering.LDAModel;
@@ -30,7 +28,7 @@ import org.apache.spark.mllib.linalg.VectorUDT;
import org.apache.spark.mllib.linalg.Vectors;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.catalyst.expressions.GenericRow;
import org.apache.spark.sql.types.Metadata;
import org.apache.spark.sql.types.StructField;
@@ -67,15 +65,13 @@ public class JavaLDAExample {
String inputFile = "data/mllib/sample_lda_data.txt";
// Parses the arguments
- SparkConf conf = new SparkConf().setAppName("JavaLDAExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaLDAExample").getOrCreate();
// Loads data
- JavaRDD<Row> points = jsc.textFile(inputFile).map(new ParseVector());
+ JavaRDD<Row> points = spark.read().text(inputFile).javaRDD().map(new ParseVector());
StructField[] fields = {new StructField("features", new VectorUDT(), false, Metadata.empty())};
StructType schema = new StructType(fields);
- Dataset<Row> dataset = sqlContext.createDataFrame(points, schema);
+ Dataset<Row> dataset = spark.createDataFrame(points, schema);
// Trains a LDA model
LDA lda = new LDA()
@@ -91,7 +87,7 @@ public class JavaLDAExample {
topics.show(false);
model.transform(dataset).show(false);
- jsc.stop();
+ spark.stop();
}
// $example off$
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaLinearRegressionWithElasticNetExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaLinearRegressionWithElasticNetExample.java
index 08fce89359..b6ea1fed25 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaLinearRegressionWithElasticNetExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaLinearRegressionWithElasticNetExample.java
@@ -17,8 +17,6 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
// $example on$
import org.apache.spark.ml.regression.LinearRegression;
import org.apache.spark.ml.regression.LinearRegressionModel;
@@ -26,18 +24,17 @@ import org.apache.spark.ml.regression.LinearRegressionTrainingSummary;
import org.apache.spark.mllib.linalg.Vectors;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example off$
public class JavaLinearRegressionWithElasticNetExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaLinearRegressionWithElasticNetExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession
+ .builder().appName("JavaLinearRegressionWithElasticNetExample").getOrCreate();
// $example on$
// Load training data
- Dataset<Row> training = sqlContext.read().format("libsvm")
+ Dataset<Row> training = spark.read().format("libsvm")
.load("data/mllib/sample_linear_regression_data.txt");
LinearRegression lr = new LinearRegression()
@@ -61,6 +58,6 @@ public class JavaLinearRegressionWithElasticNetExample {
System.out.println("r2: " + trainingSummary.r2());
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaLogisticRegressionSummaryExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaLogisticRegressionSummaryExample.java
index 73b028fb44..fd040aead4 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaLogisticRegressionSummaryExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaLogisticRegressionSummaryExample.java
@@ -17,8 +17,6 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
// $example on$
import org.apache.spark.ml.classification.BinaryLogisticRegressionSummary;
import org.apache.spark.ml.classification.LogisticRegression;
@@ -26,18 +24,17 @@ import org.apache.spark.ml.classification.LogisticRegressionModel;
import org.apache.spark.ml.classification.LogisticRegressionTrainingSummary;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.functions;
// $example off$
public class JavaLogisticRegressionSummaryExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaLogisticRegressionSummaryExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession
+ .builder().appName("JavaLogisticRegressionSummaryExample").getOrCreate();
// Load training data
- Dataset<Row> training = sqlContext.read().format("libsvm")
+ Dataset<Row> training = spark.read().format("libsvm")
.load("data/mllib/sample_libsvm_data.txt");
LogisticRegression lr = new LogisticRegression()
@@ -80,6 +77,6 @@ public class JavaLogisticRegressionSummaryExample {
lrModel.setThreshold(bestThreshold);
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaLogisticRegressionWithElasticNetExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaLogisticRegressionWithElasticNetExample.java
index 6911668522..f00c7a05cd 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaLogisticRegressionWithElasticNetExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaLogisticRegressionWithElasticNetExample.java
@@ -17,25 +17,22 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
// $example on$
import org.apache.spark.ml.classification.LogisticRegression;
import org.apache.spark.ml.classification.LogisticRegressionModel;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example off$
public class JavaLogisticRegressionWithElasticNetExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaLogisticRegressionWithElasticNetExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession
+ .builder().appName("JavaLogisticRegressionWithElasticNetExample").getOrCreate();
// $example on$
// Load training data
- Dataset<Row> training = sqlContext.read().format("libsvm")
+ Dataset<Row> training = spark.read().format("libsvm")
.load("data/mllib/sample_libsvm_data.txt");
LogisticRegression lr = new LogisticRegression()
@@ -51,6 +48,6 @@ public class JavaLogisticRegressionWithElasticNetExample {
+ lrModel.coefficients() + " Intercept: " + lrModel.intercept());
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaMaxAbsScalerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaMaxAbsScalerExample.java
index a2a072b253..80cdd364b9 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaMaxAbsScalerExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaMaxAbsScalerExample.java
@@ -17,25 +17,21 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
// $example on$
import org.apache.spark.ml.feature.MaxAbsScaler;
import org.apache.spark.ml.feature.MaxAbsScalerModel;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
// $example off$
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
public class JavaMaxAbsScalerExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaMaxAbsScalerExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext jsql = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaMaxAbsScalerExample").getOrCreate();
// $example on$
- Dataset<Row> dataFrame = jsql.read().format("libsvm").load("data/mllib/sample_libsvm_data.txt");
+ Dataset<Row> dataFrame = spark.read().format("libsvm").load("data/mllib/sample_libsvm_data.txt");
MaxAbsScaler scaler = new MaxAbsScaler()
.setInputCol("features")
.setOutputCol("scaledFeatures");
@@ -47,7 +43,7 @@ public class JavaMaxAbsScalerExample {
Dataset<Row> scaledData = scalerModel.transform(dataFrame);
scaledData.show();
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaMinMaxScalerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaMinMaxScalerExample.java
index 4aee18eeab..022940fd1e 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaMinMaxScalerExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaMinMaxScalerExample.java
@@ -17,9 +17,7 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example on$
import org.apache.spark.ml.feature.MinMaxScaler;
@@ -30,12 +28,10 @@ import org.apache.spark.sql.Row;
public class JavaMinMaxScalerExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JaveMinMaxScalerExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext jsql = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaMinMaxScalerExample").getOrCreate();
// $example on$
- Dataset<Row> dataFrame = jsql.read().format("libsvm").load("data/mllib/sample_libsvm_data.txt");
+ Dataset<Row> dataFrame = spark.read().format("libsvm").load("data/mllib/sample_libsvm_data.txt");
MinMaxScaler scaler = new MinMaxScaler()
.setInputCol("features")
.setOutputCol("scaledFeatures");
@@ -47,6 +43,6 @@ public class JavaMinMaxScalerExample {
Dataset<Row> scaledData = scalerModel.transform(dataFrame);
scaledData.show();
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaModelSelectionViaCrossValidationExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaModelSelectionViaCrossValidationExample.java
index c4122d1247..a4ec4f5815 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaModelSelectionViaCrossValidationExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaModelSelectionViaCrossValidationExample.java
@@ -21,8 +21,6 @@ package org.apache.spark.examples.ml;
import java.util.Arrays;
// $example off$
-import org.apache.spark.SparkConf;
-import org.apache.spark.SparkContext;
// $example on$
import org.apache.spark.ml.Pipeline;
import org.apache.spark.ml.PipelineStage;
@@ -37,21 +35,19 @@ import org.apache.spark.ml.tuning.ParamGridBuilder;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
// $example off$
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
/**
* Java example for Model Selection via Cross Validation.
*/
public class JavaModelSelectionViaCrossValidationExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf()
- .setAppName("JavaModelSelectionViaCrossValidationExample");
- SparkContext sc = new SparkContext(conf);
- SQLContext sqlContext = new SQLContext(sc);
+ SparkSession spark = SparkSession
+ .builder().appName("JavaModelSelectionViaCrossValidationExample").getOrCreate();
// $example on$
// Prepare training documents, which are labeled.
- Dataset<Row> training = sqlContext.createDataFrame(Arrays.asList(
+ Dataset<Row> training = spark.createDataFrame(Arrays.asList(
new JavaLabeledDocument(0L, "a b c d e spark", 1.0),
new JavaLabeledDocument(1L, "b d", 0.0),
new JavaLabeledDocument(2L,"spark f g h", 1.0),
@@ -102,7 +98,7 @@ public class JavaModelSelectionViaCrossValidationExample {
CrossValidatorModel cvModel = cv.fit(training);
// Prepare test documents, which are unlabeled.
- Dataset<Row> test = sqlContext.createDataFrame(Arrays.asList(
+ Dataset<Row> test = spark.createDataFrame(Arrays.asList(
new JavaDocument(4L, "spark i j k"),
new JavaDocument(5L, "l m n"),
new JavaDocument(6L, "mapreduce spark"),
@@ -117,6 +113,6 @@ public class JavaModelSelectionViaCrossValidationExample {
}
// $example off$
- sc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaModelSelectionViaTrainValidationSplitExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaModelSelectionViaTrainValidationSplitExample.java
index 4994f8f9fa..63a0ad1cb8 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaModelSelectionViaTrainValidationSplitExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaModelSelectionViaTrainValidationSplitExample.java
@@ -17,8 +17,6 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.SparkContext;
// $example on$
import org.apache.spark.ml.evaluation.RegressionEvaluator;
import org.apache.spark.ml.param.ParamMap;
@@ -29,7 +27,7 @@ import org.apache.spark.ml.tuning.TrainValidationSplitModel;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
// $example off$
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
/**
* Java example demonstrating model selection using TrainValidationSplit.
@@ -44,13 +42,11 @@ import org.apache.spark.sql.SQLContext;
*/
public class JavaModelSelectionViaTrainValidationSplitExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf()
- .setAppName("JavaModelSelectionViaTrainValidationSplitExample");
- SparkContext sc = new SparkContext(conf);
- SQLContext jsql = new SQLContext(sc);
+ SparkSession spark = SparkSession
+ .builder().appName("JavaModelSelectionViaTrainValidationSplitExample").getOrCreate();
// $example on$
- Dataset<Row> data = jsql.read().format("libsvm")
+ Dataset<Row> data = spark.read().format("libsvm")
.load("data/mllib/sample_linear_regression_data.txt");
// Prepare training and test data.
@@ -87,6 +83,6 @@ public class JavaModelSelectionViaTrainValidationSplitExample {
.show();
// $example off$
- sc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaMultilayerPerceptronClassifierExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaMultilayerPerceptronClassifierExample.java
index 0ca528d8cd..d547a2a64b 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaMultilayerPerceptronClassifierExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaMultilayerPerceptronClassifierExample.java
@@ -18,11 +18,9 @@
package org.apache.spark.examples.ml;
// $example on$
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
import org.apache.spark.ml.classification.MultilayerPerceptronClassificationModel;
import org.apache.spark.ml.classification.MultilayerPerceptronClassifier;
import org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator;
@@ -34,14 +32,13 @@ import org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator;
public class JavaMultilayerPerceptronClassifierExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaMultilayerPerceptronClassifierExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext jsql = new SQLContext(jsc);
+ SparkSession spark = SparkSession
+ .builder().appName("JavaMultilayerPerceptronClassifierExample").getOrCreate();
// $example on$
// Load training data
String path = "data/mllib/sample_multiclass_classification_data.txt";
- Dataset<Row> dataFrame = jsql.read().format("libsvm").load(path);
+ Dataset<Row> dataFrame = spark.read().format("libsvm").load(path);
// Split the data into train and test
Dataset<Row>[] splits = dataFrame.randomSplit(new double[]{0.6, 0.4}, 1234L);
Dataset<Row> train = splits[0];
@@ -66,6 +63,6 @@ public class JavaMultilayerPerceptronClassifierExample {
System.out.println("Precision = " + evaluator.evaluate(predictionAndLabels));
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaNGramExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaNGramExample.java
index 608bd80285..325b7b5874 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaNGramExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaNGramExample.java
@@ -17,15 +17,13 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example on$
import java.util.Arrays;
+import java.util.List;
-import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.ml.feature.NGram;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
@@ -37,16 +35,14 @@ import org.apache.spark.sql.types.StructType;
public class JavaNGramExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaNGramExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaNGramExample").getOrCreate();
// $example on$
- JavaRDD<Row> jrdd = jsc.parallelize(Arrays.asList(
+ List<Row> data = Arrays.asList(
RowFactory.create(0.0, Arrays.asList("Hi", "I", "heard", "about", "Spark")),
RowFactory.create(1.0, Arrays.asList("I", "wish", "Java", "could", "use", "case", "classes")),
RowFactory.create(2.0, Arrays.asList("Logistic", "regression", "models", "are", "neat"))
- ));
+ );
StructType schema = new StructType(new StructField[]{
new StructField("label", DataTypes.DoubleType, false, Metadata.empty()),
@@ -54,7 +50,7 @@ public class JavaNGramExample {
"words", DataTypes.createArrayType(DataTypes.StringType), false, Metadata.empty())
});
- Dataset<Row> wordDataFrame = sqlContext.createDataFrame(jrdd, schema);
+ Dataset<Row> wordDataFrame = spark.createDataFrame(data, schema);
NGram ngramTransformer = new NGram().setInputCol("words").setOutputCol("ngrams");
@@ -66,6 +62,6 @@ public class JavaNGramExample {
System.out.println();
}
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaNaiveBayesExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaNaiveBayesExample.java
index 41d7ad75b9..1f24a23609 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaNaiveBayesExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaNaiveBayesExample.java
@@ -17,16 +17,13 @@
package org.apache.spark.examples.ml;
-
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
// $example on$
import org.apache.spark.ml.classification.NaiveBayes;
import org.apache.spark.ml.classification.NaiveBayesModel;
import org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example off$
/**
@@ -35,13 +32,12 @@ import org.apache.spark.sql.SQLContext;
public class JavaNaiveBayesExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaNaiveBayesExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext jsql = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaNaiveBayesExample").getOrCreate();
// $example on$
// Load training data
- Dataset<Row> dataFrame = jsql.read().format("libsvm").load("data/mllib/sample_libsvm_data.txt");
+ Dataset<Row> dataFrame =
+ spark.read().format("libsvm").load("data/mllib/sample_libsvm_data.txt");
// Split the data into train and test
Dataset<Row>[] splits = dataFrame.randomSplit(new double[]{0.6, 0.4}, 1234L);
Dataset<Row> train = splits[0];
@@ -59,6 +55,6 @@ public class JavaNaiveBayesExample {
System.out.println("Precision = " + evaluator.evaluate(predictionAndLabels));
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaNormalizerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaNormalizerExample.java
index 31cd752136..4b3a718ea9 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaNormalizerExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaNormalizerExample.java
@@ -17,9 +17,7 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example on$
import org.apache.spark.ml.feature.Normalizer;
@@ -29,12 +27,11 @@ import org.apache.spark.sql.Row;
public class JavaNormalizerExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaNormalizerExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext jsql = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaNormalizerExample").getOrCreate();
// $example on$
- Dataset<Row> dataFrame = jsql.read().format("libsvm").load("data/mllib/sample_libsvm_data.txt");
+ Dataset<Row> dataFrame =
+ spark.read().format("libsvm").load("data/mllib/sample_libsvm_data.txt");
// Normalize each Vector using $L^1$ norm.
Normalizer normalizer = new Normalizer()
@@ -50,6 +47,6 @@ public class JavaNormalizerExample {
normalizer.transform(dataFrame, normalizer.p().w(Double.POSITIVE_INFINITY));
lInfNormData.show();
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaOneHotEncoderExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaOneHotEncoderExample.java
index 882438ca28..d6e4d21ead 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaOneHotEncoderExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaOneHotEncoderExample.java
@@ -17,14 +17,12 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example on$
import java.util.Arrays;
+import java.util.List;
-import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.ml.feature.OneHotEncoder;
import org.apache.spark.ml.feature.StringIndexer;
import org.apache.spark.ml.feature.StringIndexerModel;
@@ -39,26 +37,24 @@ import org.apache.spark.sql.types.StructType;
public class JavaOneHotEncoderExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaOneHotEncoderExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaOneHotEncoderExample").getOrCreate();
// $example on$
- JavaRDD<Row> jrdd = jsc.parallelize(Arrays.asList(
+ List<Row> data = Arrays.asList(
RowFactory.create(0, "a"),
RowFactory.create(1, "b"),
RowFactory.create(2, "c"),
RowFactory.create(3, "a"),
RowFactory.create(4, "a"),
RowFactory.create(5, "c")
- ));
+ );
StructType schema = new StructType(new StructField[]{
new StructField("id", DataTypes.DoubleType, false, Metadata.empty()),
new StructField("category", DataTypes.StringType, false, Metadata.empty())
});
- Dataset<Row> df = sqlContext.createDataFrame(jrdd, schema);
+ Dataset<Row> df = spark.createDataFrame(data, schema);
StringIndexerModel indexer = new StringIndexer()
.setInputCol("category")
@@ -72,7 +68,7 @@ public class JavaOneHotEncoderExample {
Dataset<Row> encoded = encoder.transform(indexed);
encoded.select("id", "categoryVec").show();
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaOneVsRestExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaOneVsRestExample.java
index 1f13b48bf8..9cc983bd11 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaOneVsRestExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaOneVsRestExample.java
@@ -19,8 +19,6 @@ package org.apache.spark.examples.ml;
import org.apache.commons.cli.*;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
// $example on$
import org.apache.spark.ml.classification.LogisticRegression;
import org.apache.spark.ml.classification.OneVsRest;
@@ -31,7 +29,7 @@ import org.apache.spark.mllib.linalg.Matrix;
import org.apache.spark.mllib.linalg.Vector;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.types.StructField;
// $example off$
@@ -60,9 +58,7 @@ public class JavaOneVsRestExample {
public static void main(String[] args) {
// parse the arguments
Params params = parse(args);
- SparkConf conf = new SparkConf().setAppName("JavaOneVsRestExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext jsql = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaOneVsRestExample").getOrCreate();
// $example on$
// configure the base classifier
@@ -82,7 +78,7 @@ public class JavaOneVsRestExample {
OneVsRest ovr = new OneVsRest().setClassifier(classifier);
String input = params.input;
- Dataset<Row> inputData = jsql.read().format("libsvm").load(input);
+ Dataset<Row> inputData = spark.read().format("libsvm").load(input);
Dataset<Row> train;
Dataset<Row> test;
@@ -92,7 +88,7 @@ public class JavaOneVsRestExample {
train = inputData;
// compute the number of features in the training set.
int numFeatures = inputData.first().<Vector>getAs(1).size();
- test = jsql.read().format("libsvm").option("numFeatures",
+ test = spark.read().format("libsvm").option("numFeatures",
String.valueOf(numFeatures)).load(testInput);
} else {
double f = params.fracTest;
@@ -131,7 +127,7 @@ public class JavaOneVsRestExample {
System.out.println(results);
// $example off$
- jsc.stop();
+ spark.stop();
}
private static Params parse(String[] args) {
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaPCAExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaPCAExample.java
index a792fd7d47..6b1dcb68ba 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaPCAExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaPCAExample.java
@@ -17,14 +17,12 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example on$
import java.util.Arrays;
+import java.util.List;
-import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.ml.feature.PCA;
import org.apache.spark.ml.feature.PCAModel;
import org.apache.spark.mllib.linalg.VectorUDT;
@@ -39,22 +37,20 @@ import org.apache.spark.sql.types.StructType;
public class JavaPCAExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaPCAExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext jsql = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaPCAExample").getOrCreate();
// $example on$
- JavaRDD<Row> data = jsc.parallelize(Arrays.asList(
+ List<Row> data = Arrays.asList(
RowFactory.create(Vectors.sparse(5, new int[]{1, 3}, new double[]{1.0, 7.0})),
RowFactory.create(Vectors.dense(2.0, 0.0, 3.0, 4.0, 5.0)),
RowFactory.create(Vectors.dense(4.0, 0.0, 0.0, 6.0, 7.0))
- ));
+ );
StructType schema = new StructType(new StructField[]{
new StructField("features", new VectorUDT(), false, Metadata.empty()),
});
- Dataset<Row> df = jsql.createDataFrame(data, schema);
+ Dataset<Row> df = spark.createDataFrame(data, schema);
PCAModel pca = new PCA()
.setInputCol("features")
@@ -65,7 +61,7 @@ public class JavaPCAExample {
Dataset<Row> result = pca.transform(df).select("pcaFeatures");
result.show();
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaPipelineExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaPipelineExample.java
index 305420f208..556a457326 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaPipelineExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaPipelineExample.java
@@ -19,11 +19,7 @@ package org.apache.spark.examples.ml;
// $example on$
import java.util.Arrays;
-// $example off$
-import org.apache.spark.SparkConf;
-import org.apache.spark.SparkContext;
-// $example on$
import org.apache.spark.ml.Pipeline;
import org.apache.spark.ml.PipelineModel;
import org.apache.spark.ml.PipelineStage;
@@ -33,20 +29,18 @@ import org.apache.spark.ml.feature.Tokenizer;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
// $example off$
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
/**
* Java example for simple text document 'Pipeline'.
*/
public class JavaPipelineExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaPipelineExample");
- SparkContext sc = new SparkContext(conf);
- SQLContext sqlContext = new SQLContext(sc);
+ SparkSession spark = SparkSession.builder().appName("JavaPipelineExample").getOrCreate();
// $example on$
// Prepare training documents, which are labeled.
- Dataset<Row> training = sqlContext.createDataFrame(Arrays.asList(
+ Dataset<Row> training = spark.createDataFrame(Arrays.asList(
new JavaLabeledDocument(0L, "a b c d e spark", 1.0),
new JavaLabeledDocument(1L, "b d", 0.0),
new JavaLabeledDocument(2L, "spark f g h", 1.0),
@@ -71,7 +65,7 @@ public class JavaPipelineExample {
PipelineModel model = pipeline.fit(training);
// Prepare test documents, which are unlabeled.
- Dataset<Row> test = sqlContext.createDataFrame(Arrays.asList(
+ Dataset<Row> test = spark.createDataFrame(Arrays.asList(
new JavaDocument(4L, "spark i j k"),
new JavaDocument(5L, "l m n"),
new JavaDocument(6L, "mapreduce spark"),
@@ -86,6 +80,6 @@ public class JavaPipelineExample {
}
// $example off$
- sc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaPolynomialExpansionExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaPolynomialExpansionExample.java
index 48fc3c8acb..e328454c70 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaPolynomialExpansionExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaPolynomialExpansionExample.java
@@ -17,15 +17,12 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example on$
import java.util.Arrays;
import java.util.List;
-import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.ml.feature.PolynomialExpansion;
import org.apache.spark.mllib.linalg.VectorUDT;
import org.apache.spark.mllib.linalg.Vectors;
@@ -39,9 +36,7 @@ import org.apache.spark.sql.types.StructType;
public class JavaPolynomialExpansionExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaPolynomialExpansionExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext jsql = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaPolynomialExpansionExample").getOrCreate();
// $example on$
PolynomialExpansion polyExpansion = new PolynomialExpansion()
@@ -49,17 +44,17 @@ public class JavaPolynomialExpansionExample {
.setOutputCol("polyFeatures")
.setDegree(3);
- JavaRDD<Row> data = jsc.parallelize(Arrays.asList(
+ List<Row> data = Arrays.asList(
RowFactory.create(Vectors.dense(-2.0, 2.3)),
RowFactory.create(Vectors.dense(0.0, 0.0)),
RowFactory.create(Vectors.dense(0.6, -1.1))
- ));
+ );
StructType schema = new StructType(new StructField[]{
new StructField("features", new VectorUDT(), false, Metadata.empty()),
});
- Dataset<Row> df = jsql.createDataFrame(data, schema);
+ Dataset<Row> df = spark.createDataFrame(data, schema);
Dataset<Row> polyDF = polyExpansion.transform(df);
List<Row> rows = polyDF.select("polyFeatures").takeAsList(3);
@@ -67,6 +62,6 @@ public class JavaPolynomialExpansionExample {
System.out.println(r.get(0));
}
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaQuantileDiscretizerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaQuantileDiscretizerExample.java
index 7b226fede9..94e3fafcab 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaQuantileDiscretizerExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaQuantileDiscretizerExample.java
@@ -17,13 +17,11 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example on$
import java.util.Arrays;
+import java.util.List;
-import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.ml.feature.QuantileDiscretizer;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
@@ -36,19 +34,16 @@ import org.apache.spark.sql.types.StructType;
public class JavaQuantileDiscretizerExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaQuantileDiscretizerExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession
+ .builder().appName("JavaQuantileDiscretizerExample").getOrCreate();
// $example on$
- JavaRDD<Row> jrdd = jsc.parallelize(
- Arrays.asList(
- RowFactory.create(0, 18.0),
- RowFactory.create(1, 19.0),
- RowFactory.create(2, 8.0),
- RowFactory.create(3, 5.0),
- RowFactory.create(4, 2.2)
- )
+ List<Row> data = Arrays.asList(
+ RowFactory.create(0, 18.0),
+ RowFactory.create(1, 19.0),
+ RowFactory.create(2, 8.0),
+ RowFactory.create(3, 5.0),
+ RowFactory.create(4, 2.2)
);
StructType schema = new StructType(new StructField[]{
@@ -56,7 +51,7 @@ public class JavaQuantileDiscretizerExample {
new StructField("hour", DataTypes.DoubleType, false, Metadata.empty())
});
- Dataset<Row> df = sqlContext.createDataFrame(jrdd, schema);
+ Dataset<Row> df = spark.createDataFrame(data, schema);
QuantileDiscretizer discretizer = new QuantileDiscretizer()
.setInputCol("hour")
@@ -66,6 +61,6 @@ public class JavaQuantileDiscretizerExample {
Dataset<Row> result = discretizer.fit(df).transform(df);
result.show();
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaRFormulaExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaRFormulaExample.java
index 8c453bf80d..8282ce01d3 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaRFormulaExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaRFormulaExample.java
@@ -17,14 +17,12 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example on$
import java.util.Arrays;
+import java.util.List;
-import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.ml.feature.RFormula;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
@@ -37,9 +35,7 @@ import static org.apache.spark.sql.types.DataTypes.*;
public class JavaRFormulaExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaRFormulaExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaRFormulaExample").getOrCreate();
// $example on$
StructType schema = createStructType(new StructField[]{
@@ -49,13 +45,13 @@ public class JavaRFormulaExample {
createStructField("clicked", DoubleType, false)
});
- JavaRDD<Row> rdd = jsc.parallelize(Arrays.asList(
+ List<Row> data = Arrays.asList(
RowFactory.create(7, "US", 18, 1.0),
RowFactory.create(8, "CA", 12, 0.0),
RowFactory.create(9, "NZ", 15, 0.0)
- ));
+ );
- Dataset<Row> dataset = sqlContext.createDataFrame(rdd, schema);
+ Dataset<Row> dataset = spark.createDataFrame(data, schema);
RFormula formula = new RFormula()
.setFormula("clicked ~ country + hour")
.setFeaturesCol("features")
@@ -63,7 +59,7 @@ public class JavaRFormulaExample {
Dataset<Row> output = formula.fit(dataset).transform(dataset);
output.select("features", "label").show();
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaRandomForestClassifierExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaRandomForestClassifierExample.java
index 05c2bc9622..21e783a968 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaRandomForestClassifierExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaRandomForestClassifierExample.java
@@ -17,8 +17,6 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
// $example on$
import org.apache.spark.ml.Pipeline;
import org.apache.spark.ml.PipelineModel;
@@ -29,19 +27,17 @@ import org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator;
import org.apache.spark.ml.feature.*;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example off$
public class JavaRandomForestClassifierExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaRandomForestClassifierExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession
+ .builder().appName("JavaRandomForestClassifierExample").getOrCreate();
// $example on$
// Load and parse the data file, converting it to a DataFrame.
- Dataset<Row> data =
- sqlContext.read().format("libsvm").load("data/mllib/sample_libsvm_data.txt");
+ Dataset<Row> data = spark.read().format("libsvm").load("data/mllib/sample_libsvm_data.txt");
// Index labels, adding metadata to the label column.
// Fit on whole dataset to include all labels in index.
@@ -98,6 +94,6 @@ public class JavaRandomForestClassifierExample {
System.out.println("Learned classification forest model:\n" + rfModel.toDebugString());
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaRandomForestRegressorExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaRandomForestRegressorExample.java
index d366967083..ece184a878 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaRandomForestRegressorExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaRandomForestRegressorExample.java
@@ -17,8 +17,6 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
// $example on$
import org.apache.spark.ml.Pipeline;
import org.apache.spark.ml.PipelineModel;
@@ -30,19 +28,17 @@ import org.apache.spark.ml.regression.RandomForestRegressionModel;
import org.apache.spark.ml.regression.RandomForestRegressor;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example off$
public class JavaRandomForestRegressorExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaRandomForestRegressorExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession
+ .builder().appName("JavaRandomForestRegressorExample").getOrCreate();
// $example on$
// Load and parse the data file, converting it to a DataFrame.
- Dataset<Row> data =
- sqlContext.read().format("libsvm").load("data/mllib/sample_libsvm_data.txt");
+ Dataset<Row> data = spark.read().format("libsvm").load("data/mllib/sample_libsvm_data.txt");
// Automatically identify categorical features, and index them.
// Set maxCategories so features with > 4 distinct values are treated as continuous.
@@ -87,6 +83,6 @@ public class JavaRandomForestRegressorExample {
System.out.println("Learned regression forest model:\n" + rfModel.toDebugString());
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaSQLTransformerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaSQLTransformerExample.java
index 7e3ca99d7c..492718bbdb 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaSQLTransformerExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaSQLTransformerExample.java
@@ -19,36 +19,31 @@ package org.apache.spark.examples.ml;
// $example on$
import java.util.Arrays;
+import java.util.List;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaRDD;
-import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.ml.feature.SQLTransformer;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.types.*;
// $example off$
public class JavaSQLTransformerExample {
public static void main(String[] args) {
-
- SparkConf conf = new SparkConf().setAppName("JavaSQLTransformerExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaSQLTransformerExample").getOrCreate();
// $example on$
- JavaRDD<Row> jrdd = jsc.parallelize(Arrays.asList(
+ List<Row> data = Arrays.asList(
RowFactory.create(0, 1.0, 3.0),
RowFactory.create(2, 2.0, 5.0)
- ));
+ );
StructType schema = new StructType(new StructField [] {
new StructField("id", DataTypes.IntegerType, false, Metadata.empty()),
new StructField("v1", DataTypes.DoubleType, false, Metadata.empty()),
new StructField("v2", DataTypes.DoubleType, false, Metadata.empty())
});
- Dataset<Row> df = sqlContext.createDataFrame(jrdd, schema);
+ Dataset<Row> df = spark.createDataFrame(data, schema);
SQLTransformer sqlTrans = new SQLTransformer().setStatement(
"SELECT *, (v1 + v2) AS v3, (v1 * v2) AS v4 FROM __THIS__");
@@ -56,6 +51,6 @@ public class JavaSQLTransformerExample {
sqlTrans.transform(df).show();
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaSimpleParamsExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaSimpleParamsExample.java
index cb911ef5ef..f906843640 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaSimpleParamsExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaSimpleParamsExample.java
@@ -21,8 +21,6 @@ import java.util.List;
import com.google.common.collect.Lists;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.ml.classification.LogisticRegressionModel;
import org.apache.spark.ml.param.ParamMap;
import org.apache.spark.ml.classification.LogisticRegression;
@@ -30,7 +28,7 @@ import org.apache.spark.mllib.linalg.Vectors;
import org.apache.spark.mllib.regression.LabeledPoint;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
/**
* A simple example demonstrating ways to specify parameters for Estimators and Transformers.
@@ -42,9 +40,7 @@ import org.apache.spark.sql.SQLContext;
public class JavaSimpleParamsExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaSimpleParamsExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext jsql = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaSimpleParamsExample").getOrCreate();
// Prepare training data.
// We use LabeledPoint, which is a JavaBean. Spark SQL can convert RDDs of JavaBeans
@@ -55,7 +51,7 @@ public class JavaSimpleParamsExample {
new LabeledPoint(0.0, Vectors.dense(2.0, 1.3, 1.0)),
new LabeledPoint(1.0, Vectors.dense(0.0, 1.2, -0.5)));
Dataset<Row> training =
- jsql.createDataFrame(jsc.parallelize(localTraining), LabeledPoint.class);
+ spark.createDataFrame(localTraining, LabeledPoint.class);
// Create a LogisticRegression instance. This instance is an Estimator.
LogisticRegression lr = new LogisticRegression();
@@ -96,7 +92,7 @@ public class JavaSimpleParamsExample {
new LabeledPoint(1.0, Vectors.dense(-1.0, 1.5, 1.3)),
new LabeledPoint(0.0, Vectors.dense(3.0, 2.0, -0.1)),
new LabeledPoint(1.0, Vectors.dense(0.0, 2.2, -1.5)));
- Dataset<Row> test = jsql.createDataFrame(jsc.parallelize(localTest), LabeledPoint.class);
+ Dataset<Row> test = spark.createDataFrame(localTest, LabeledPoint.class);
// Make predictions on test documents using the Transformer.transform() method.
// LogisticRegressionModel.transform will only use the 'features' column.
@@ -109,6 +105,6 @@ public class JavaSimpleParamsExample {
+ ", prediction=" + r.get(3));
}
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaSimpleTextClassificationPipeline.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaSimpleTextClassificationPipeline.java
index a18a60f448..9516ce1f4f 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaSimpleTextClassificationPipeline.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaSimpleTextClassificationPipeline.java
@@ -21,8 +21,6 @@ import java.util.List;
import com.google.common.collect.Lists;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.ml.Pipeline;
import org.apache.spark.ml.PipelineModel;
import org.apache.spark.ml.PipelineStage;
@@ -31,7 +29,7 @@ import org.apache.spark.ml.feature.HashingTF;
import org.apache.spark.ml.feature.Tokenizer;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
/**
* A simple text classification pipeline that recognizes "spark" from input text. It uses the Java
@@ -44,9 +42,8 @@ import org.apache.spark.sql.SQLContext;
public class JavaSimpleTextClassificationPipeline {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaSimpleTextClassificationPipeline");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext jsql = new SQLContext(jsc);
+ SparkSession spark = SparkSession
+ .builder().appName("JavaSimpleTextClassificationPipeline").getOrCreate();
// Prepare training documents, which are labeled.
List<LabeledDocument> localTraining = Lists.newArrayList(
@@ -55,7 +52,7 @@ public class JavaSimpleTextClassificationPipeline {
new LabeledDocument(2L, "spark f g h", 1.0),
new LabeledDocument(3L, "hadoop mapreduce", 0.0));
Dataset<Row> training =
- jsql.createDataFrame(jsc.parallelize(localTraining), LabeledDocument.class);
+ spark.createDataFrame(localTraining, LabeledDocument.class);
// Configure an ML pipeline, which consists of three stages: tokenizer, hashingTF, and lr.
Tokenizer tokenizer = new Tokenizer()
@@ -80,7 +77,7 @@ public class JavaSimpleTextClassificationPipeline {
new Document(5L, "l m n"),
new Document(6L, "spark hadoop spark"),
new Document(7L, "apache hadoop"));
- Dataset<Row> test = jsql.createDataFrame(jsc.parallelize(localTest), Document.class);
+ Dataset<Row> test = spark.createDataFrame(localTest, Document.class);
// Make predictions on test documents.
Dataset<Row> predictions = model.transform(test);
@@ -89,6 +86,6 @@ public class JavaSimpleTextClassificationPipeline {
+ ", prediction=" + r.get(3));
}
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaStandardScalerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaStandardScalerExample.java
index e2dd759c0a..10f82f2233 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaStandardScalerExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaStandardScalerExample.java
@@ -17,9 +17,7 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example on$
import org.apache.spark.ml.feature.StandardScaler;
@@ -30,12 +28,11 @@ import org.apache.spark.sql.Row;
public class JavaStandardScalerExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaStandardScalerExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext jsql = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaStandardScalerExample").getOrCreate();
// $example on$
- Dataset<Row> dataFrame = jsql.read().format("libsvm").load("data/mllib/sample_libsvm_data.txt");
+ Dataset<Row> dataFrame =
+ spark.read().format("libsvm").load("data/mllib/sample_libsvm_data.txt");
StandardScaler scaler = new StandardScaler()
.setInputCol("features")
@@ -50,6 +47,6 @@ public class JavaStandardScalerExample {
Dataset<Row> scaledData = scalerModel.transform(dataFrame);
scaledData.show();
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaStopWordsRemoverExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaStopWordsRemoverExample.java
index 0ff3782cb3..23ed071c9f 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaStopWordsRemoverExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaStopWordsRemoverExample.java
@@ -17,14 +17,12 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example on$
import java.util.Arrays;
+import java.util.List;
-import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.ml.feature.StopWordsRemover;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
@@ -38,28 +36,26 @@ import org.apache.spark.sql.types.StructType;
public class JavaStopWordsRemoverExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaStopWordsRemoverExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext jsql = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaStopWordsRemoverExample").getOrCreate();
// $example on$
StopWordsRemover remover = new StopWordsRemover()
.setInputCol("raw")
.setOutputCol("filtered");
- JavaRDD<Row> rdd = jsc.parallelize(Arrays.asList(
+ List<Row> data = Arrays.asList(
RowFactory.create(Arrays.asList("I", "saw", "the", "red", "baloon")),
RowFactory.create(Arrays.asList("Mary", "had", "a", "little", "lamb"))
- ));
+ );
StructType schema = new StructType(new StructField[]{
new StructField(
"raw", DataTypes.createArrayType(DataTypes.StringType), false, Metadata.empty())
});
- Dataset<Row> dataset = jsql.createDataFrame(rdd, schema);
+ Dataset<Row> dataset = spark.createDataFrame(data, schema);
remover.transform(dataset).show();
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaStringIndexerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaStringIndexerExample.java
index ceacbb4fb3..d4c2cf96a7 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaStringIndexerExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaStringIndexerExample.java
@@ -17,14 +17,12 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example on$
import java.util.Arrays;
+import java.util.List;
-import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.ml.feature.StringIndexer;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
@@ -37,30 +35,28 @@ import static org.apache.spark.sql.types.DataTypes.*;
public class JavaStringIndexerExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaStringIndexerExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaStringIndexerExample").getOrCreate();
// $example on$
- JavaRDD<Row> jrdd = jsc.parallelize(Arrays.asList(
+ List<Row> data = Arrays.asList(
RowFactory.create(0, "a"),
RowFactory.create(1, "b"),
RowFactory.create(2, "c"),
RowFactory.create(3, "a"),
RowFactory.create(4, "a"),
RowFactory.create(5, "c")
- ));
+ );
StructType schema = new StructType(new StructField[]{
createStructField("id", IntegerType, false),
createStructField("category", StringType, false)
});
- Dataset<Row> df = sqlContext.createDataFrame(jrdd, schema);
+ Dataset<Row> df = spark.createDataFrame(data, schema);
StringIndexer indexer = new StringIndexer()
.setInputCol("category")
.setOutputCol("categoryIndex");
Dataset<Row> indexed = indexer.fit(df).transform(df);
indexed.show();
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaTfIdfExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaTfIdfExample.java
index 107c835f2e..a816991777 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaTfIdfExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaTfIdfExample.java
@@ -19,10 +19,8 @@ package org.apache.spark.examples.ml;
// $example on$
import java.util.Arrays;
+import java.util.List;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaRDD;
-import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.ml.feature.HashingTF;
import org.apache.spark.ml.feature.IDF;
import org.apache.spark.ml.feature.IDFModel;
@@ -31,7 +29,7 @@ import org.apache.spark.mllib.linalg.Vector;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.Metadata;
import org.apache.spark.sql.types.StructField;
@@ -40,21 +38,19 @@ import org.apache.spark.sql.types.StructType;
public class JavaTfIdfExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaTfIdfExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaTfIdfExample").getOrCreate();
// $example on$
- JavaRDD<Row> jrdd = jsc.parallelize(Arrays.asList(
+ List<Row> data = Arrays.asList(
RowFactory.create(0, "Hi I heard about Spark"),
RowFactory.create(0, "I wish Java could use case classes"),
RowFactory.create(1, "Logistic regression models are neat")
- ));
+ );
StructType schema = new StructType(new StructField[]{
new StructField("label", DataTypes.DoubleType, false, Metadata.empty()),
new StructField("sentence", DataTypes.StringType, false, Metadata.empty())
});
- Dataset<Row> sentenceData = sqlContext.createDataFrame(jrdd, schema);
+ Dataset<Row> sentenceData = spark.createDataFrame(data, schema);
Tokenizer tokenizer = new Tokenizer().setInputCol("sentence").setOutputCol("words");
Dataset<Row> wordsData = tokenizer.transform(sentenceData);
int numFeatures = 20;
@@ -76,6 +72,6 @@ public class JavaTfIdfExample {
}
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaTokenizerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaTokenizerExample.java
index 9225fe2262..a65735a5e5 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaTokenizerExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaTokenizerExample.java
@@ -17,14 +17,12 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example on$
import java.util.Arrays;
+import java.util.List;
-import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.ml.feature.RegexTokenizer;
import org.apache.spark.ml.feature.Tokenizer;
import org.apache.spark.sql.Dataset;
@@ -38,23 +36,21 @@ import org.apache.spark.sql.types.StructType;
public class JavaTokenizerExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaTokenizerExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaTokenizerExample").getOrCreate();
// $example on$
- JavaRDD<Row> jrdd = jsc.parallelize(Arrays.asList(
+ List<Row> data = Arrays.asList(
RowFactory.create(0, "Hi I heard about Spark"),
RowFactory.create(1, "I wish Java could use case classes"),
RowFactory.create(2, "Logistic,regression,models,are,neat")
- ));
+ );
StructType schema = new StructType(new StructField[]{
new StructField("label", DataTypes.IntegerType, false, Metadata.empty()),
new StructField("sentence", DataTypes.StringType, false, Metadata.empty())
});
- Dataset<Row> sentenceDataFrame = sqlContext.createDataFrame(jrdd, schema);
+ Dataset<Row> sentenceDataFrame = spark.createDataFrame(data, schema);
Tokenizer tokenizer = new Tokenizer().setInputCol("sentence").setOutputCol("words");
@@ -70,6 +66,6 @@ public class JavaTokenizerExample {
.setOutputCol("words")
.setPattern("\\W"); // alternatively .setPattern("\\w+").setGaps(false);
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorAssemblerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorAssemblerExample.java
index 953ad455b1..9569bc2412 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorAssemblerExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorAssemblerExample.java
@@ -17,14 +17,11 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example on$
import java.util.Arrays;
-import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.ml.feature.VectorAssembler;
import org.apache.spark.mllib.linalg.VectorUDT;
import org.apache.spark.mllib.linalg.Vectors;
@@ -38,9 +35,7 @@ import static org.apache.spark.sql.types.DataTypes.*;
public class JavaVectorAssemblerExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaVectorAssemblerExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaVectorAssemblerExample").getOrCreate();
// $example on$
StructType schema = createStructType(new StructField[]{
@@ -51,8 +46,7 @@ public class JavaVectorAssemblerExample {
createStructField("clicked", DoubleType, false)
});
Row row = RowFactory.create(0, 18, 1.0, Vectors.dense(0.0, 10.0, 0.5), 1.0);
- JavaRDD<Row> rdd = jsc.parallelize(Arrays.asList(row));
- Dataset<Row> dataset = sqlContext.createDataFrame(rdd, schema);
+ Dataset<Row> dataset = spark.createDataFrame(Arrays.asList(row), schema);
VectorAssembler assembler = new VectorAssembler()
.setInputCols(new String[]{"hour", "mobile", "userFeatures"})
@@ -61,7 +55,7 @@ public class JavaVectorAssemblerExample {
Dataset<Row> output = assembler.transform(dataset);
System.out.println(output.select("features", "clicked").first());
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorIndexerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorIndexerExample.java
index b3b5953ee7..217d5a06d1 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorIndexerExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorIndexerExample.java
@@ -17,9 +17,7 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example on$
import java.util.Map;
@@ -32,12 +30,10 @@ import org.apache.spark.sql.Row;
public class JavaVectorIndexerExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaVectorIndexerExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext jsql = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaVectorIndexerExample").getOrCreate();
// $example on$
- Dataset<Row> data = jsql.read().format("libsvm").load("data/mllib/sample_libsvm_data.txt");
+ Dataset<Row> data = spark.read().format("libsvm").load("data/mllib/sample_libsvm_data.txt");
VectorIndexer indexer = new VectorIndexer()
.setInputCol("features")
@@ -57,6 +53,6 @@ public class JavaVectorIndexerExample {
Dataset<Row> indexedData = indexerModel.transform(data);
indexedData.show();
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorSlicerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorSlicerExample.java
index 2ae57c3577..4f1ea824a3 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorSlicerExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorSlicerExample.java
@@ -17,14 +17,13 @@
package org.apache.spark.examples.ml;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
// $example on$
+import java.util.List;
+
import com.google.common.collect.Lists;
-import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.ml.attribute.Attribute;
import org.apache.spark.ml.attribute.AttributeGroup;
import org.apache.spark.ml.attribute.NumericAttribute;
@@ -38,9 +37,7 @@ import org.apache.spark.sql.types.*;
public class JavaVectorSlicerExample {
public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("JavaVectorSlicerExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext jsql = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaVectorSlicerExample").getOrCreate();
// $example on$
Attribute[] attrs = new Attribute[]{
@@ -50,13 +47,13 @@ public class JavaVectorSlicerExample {
};
AttributeGroup group = new AttributeGroup("userFeatures", attrs);
- JavaRDD<Row> jrdd = jsc.parallelize(Lists.newArrayList(
+ List<Row> data = Lists.newArrayList(
RowFactory.create(Vectors.sparse(3, new int[]{0, 1}, new double[]{-2.0, 2.3})),
RowFactory.create(Vectors.dense(-2.0, 2.3, 0.0))
- ));
+ );
Dataset<Row> dataset =
- jsql.createDataFrame(jrdd, (new StructType()).add(group.toStructField()));
+ spark.createDataFrame(data, (new StructType()).add(group.toStructField()));
VectorSlicer vectorSlicer = new VectorSlicer()
.setInputCol("userFeatures").setOutputCol("features");
@@ -68,7 +65,7 @@ public class JavaVectorSlicerExample {
System.out.println(output.select("userFeatures", "features").first());
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaWord2VecExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaWord2VecExample.java
index c5bb1eaaa3..d9b1a79b52 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaWord2VecExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaWord2VecExample.java
@@ -19,37 +19,32 @@ package org.apache.spark.examples.ml;
// $example on$
import java.util.Arrays;
+import java.util.List;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaRDD;
-import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.ml.feature.Word2Vec;
import org.apache.spark.ml.feature.Word2VecModel;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.types.*;
// $example off$
public class JavaWord2VecExample {
public static void main(String[] args) {
-
- SparkConf conf = new SparkConf().setAppName("JavaWord2VecExample");
- JavaSparkContext jsc = new JavaSparkContext(conf);
- SQLContext sqlContext = new SQLContext(jsc);
+ SparkSession spark = SparkSession.builder().appName("JavaWord2VecExample").getOrCreate();
// $example on$
// Input data: Each row is a bag of words from a sentence or document.
- JavaRDD<Row> jrdd = jsc.parallelize(Arrays.asList(
+ List<Row> data = Arrays.asList(
RowFactory.create(Arrays.asList("Hi I heard about Spark".split(" "))),
RowFactory.create(Arrays.asList("I wish Java could use case classes".split(" "))),
RowFactory.create(Arrays.asList("Logistic regression models are neat".split(" ")))
- ));
+ );
StructType schema = new StructType(new StructField[]{
new StructField("text", new ArrayType(DataTypes.StringType, true), false, Metadata.empty())
});
- Dataset<Row> documentDF = sqlContext.createDataFrame(jrdd, schema);
+ Dataset<Row> documentDF = spark.createDataFrame(data, schema);
// Learn a mapping from words to Vectors.
Word2Vec word2Vec = new Word2Vec()
@@ -64,6 +59,6 @@ public class JavaWord2VecExample {
}
// $example off$
- jsc.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/sql/JavaSparkSQL.java b/examples/src/main/java/org/apache/spark/examples/sql/JavaSparkSQL.java
index 354a5306ed..ec2142e756 100644
--- a/examples/src/main/java/org/apache/spark/examples/sql/JavaSparkSQL.java
+++ b/examples/src/main/java/org/apache/spark/examples/sql/JavaSparkSQL.java
@@ -21,14 +21,12 @@ import java.io.Serializable;
import java.util.Arrays;
import java.util.List;
-import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
-import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
public class JavaSparkSQL {
public static class Person implements Serializable {
@@ -53,13 +51,12 @@ public class JavaSparkSQL {
}
public static void main(String[] args) throws Exception {
- SparkConf sparkConf = new SparkConf().setAppName("JavaSparkSQL");
- JavaSparkContext ctx = new JavaSparkContext(sparkConf);
- SQLContext sqlContext = new SQLContext(ctx);
+ SparkSession spark = SparkSession.builder().appName("JavaSparkSQL").getOrCreate();
System.out.println("=== Data source: RDD ===");
// Load a text file and convert each line to a Java Bean.
- JavaRDD<Person> people = ctx.textFile("examples/src/main/resources/people.txt").map(
+ String file = "examples/src/main/resources/people.txt";
+ JavaRDD<Person> people = spark.read().text(file).javaRDD().map(
new Function<String, Person>() {
@Override
public Person call(String line) {
@@ -74,12 +71,11 @@ public class JavaSparkSQL {
});
// Apply a schema to an RDD of Java Beans and register it as a table.
- Dataset<Row> schemaPeople = sqlContext.createDataFrame(people, Person.class);
+ Dataset<Row> schemaPeople = spark.createDataFrame(people, Person.class);
schemaPeople.registerTempTable("people");
// SQL can be run over RDDs that have been registered as tables.
- Dataset<Row> teenagers =
- sqlContext.sql("SELECT name FROM people WHERE age >= 13 AND age <= 19");
+ Dataset<Row> teenagers = spark.sql("SELECT name FROM people WHERE age >= 13 AND age <= 19");
// The results of SQL queries are DataFrames and support all the normal RDD operations.
// The columns of a row in the result can be accessed by ordinal.
@@ -100,12 +96,12 @@ public class JavaSparkSQL {
// Read in the parquet file created above.
// Parquet files are self-describing so the schema is preserved.
// The result of loading a parquet file is also a DataFrame.
- Dataset<Row> parquetFile = sqlContext.read().parquet("people.parquet");
+ Dataset<Row> parquetFile = spark.read().parquet("people.parquet");
//Parquet files can also be registered as tables and then used in SQL statements.
parquetFile.registerTempTable("parquetFile");
Dataset<Row> teenagers2 =
- sqlContext.sql("SELECT name FROM parquetFile WHERE age >= 13 AND age <= 19");
+ spark.sql("SELECT name FROM parquetFile WHERE age >= 13 AND age <= 19");
teenagerNames = teenagers2.toJavaRDD().map(new Function<Row, String>() {
@Override
public String call(Row row) {
@@ -121,7 +117,7 @@ public class JavaSparkSQL {
// The path can be either a single text file or a directory storing text files.
String path = "examples/src/main/resources/people.json";
// Create a DataFrame from the file(s) pointed by path
- Dataset<Row> peopleFromJsonFile = sqlContext.read().json(path);
+ Dataset<Row> peopleFromJsonFile = spark.read().json(path);
// Because the schema of a JSON dataset is automatically inferred, to write queries,
// it is better to take a look at what is the schema.
@@ -135,8 +131,7 @@ public class JavaSparkSQL {
peopleFromJsonFile.registerTempTable("people");
// SQL statements can be run by using the sql methods provided by sqlContext.
- Dataset<Row> teenagers3 =
- sqlContext.sql("SELECT name FROM people WHERE age >= 13 AND age <= 19");
+ Dataset<Row> teenagers3 = spark.sql("SELECT name FROM people WHERE age >= 13 AND age <= 19");
// The results of SQL queries are DataFrame and support all the normal RDD operations.
// The columns of a row in the result can be accessed by ordinal.
@@ -152,8 +147,8 @@ public class JavaSparkSQL {
// a RDD[String] storing one JSON object per string.
List<String> jsonData = Arrays.asList(
"{\"name\":\"Yin\",\"address\":{\"city\":\"Columbus\",\"state\":\"Ohio\"}}");
- JavaRDD<String> anotherPeopleRDD = ctx.parallelize(jsonData);
- Dataset<Row> peopleFromJsonRDD = sqlContext.read().json(anotherPeopleRDD.rdd());
+ JavaRDD<String> anotherPeopleRDD = spark.createDataFrame(jsonData, String.class).toJSON().javaRDD();
+ Dataset<Row> peopleFromJsonRDD = spark.read().json(anotherPeopleRDD);
// Take a look at the schema of this new DataFrame.
peopleFromJsonRDD.printSchema();
@@ -166,7 +161,7 @@ public class JavaSparkSQL {
peopleFromJsonRDD.registerTempTable("people2");
- Dataset<Row> peopleWithCity = sqlContext.sql("SELECT name, address.city FROM people2");
+ Dataset<Row> peopleWithCity = spark.sql("SELECT name, address.city FROM people2");
List<String> nameAndCity = peopleWithCity.toJavaRDD().map(new Function<Row, String>() {
@Override
public String call(Row row) {
@@ -177,6 +172,6 @@ public class JavaSparkSQL {
System.out.println(name);
}
- ctx.stop();
+ spark.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/streaming/JavaSqlNetworkWordCount.java b/examples/src/main/java/org/apache/spark/examples/streaming/JavaSqlNetworkWordCount.java
index 7aa8862761..44f1e800fe 100644
--- a/examples/src/main/java/org/apache/spark/examples/streaming/JavaSqlNetworkWordCount.java
+++ b/examples/src/main/java/org/apache/spark/examples/streaming/JavaSqlNetworkWordCount.java
@@ -22,14 +22,13 @@ import java.util.Iterator;
import java.util.regex.Pattern;
import org.apache.spark.SparkConf;
-import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.VoidFunction2;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
import org.apache.spark.api.java.StorageLevels;
import org.apache.spark.streaming.Durations;
import org.apache.spark.streaming.Time;
@@ -82,7 +81,7 @@ public final class JavaSqlNetworkWordCount {
words.foreachRDD(new VoidFunction2<JavaRDD<String>, Time>() {
@Override
public void call(JavaRDD<String> rdd, Time time) {
- SQLContext sqlContext = JavaSQLContextSingleton.getInstance(rdd.context());
+ SparkSession spark = JavaSparkSessionSingleton.getInstance(rdd.context().getConf());
// Convert JavaRDD[String] to JavaRDD[bean class] to DataFrame
JavaRDD<JavaRecord> rowRDD = rdd.map(new Function<String, JavaRecord>() {
@@ -93,14 +92,14 @@ public final class JavaSqlNetworkWordCount {
return record;
}
});
- Dataset<Row> wordsDataFrame = sqlContext.createDataFrame(rowRDD, JavaRecord.class);
+ Dataset<Row> wordsDataFrame = spark.createDataFrame(rowRDD, JavaRecord.class);
// Register as table
wordsDataFrame.registerTempTable("words");
// Do word count on table using SQL and print it
Dataset<Row> wordCountsDataFrame =
- sqlContext.sql("select word, count(*) as total from words group by word");
+ spark.sql("select word, count(*) as total from words group by word");
System.out.println("========= " + time + "=========");
wordCountsDataFrame.show();
}
@@ -111,12 +110,12 @@ public final class JavaSqlNetworkWordCount {
}
}
-/** Lazily instantiated singleton instance of SQLContext */
-class JavaSQLContextSingleton {
- private static transient SQLContext instance = null;
- public static SQLContext getInstance(SparkContext sparkContext) {
+/** Lazily instantiated singleton instance of SparkSession */
+class JavaSparkSessionSingleton {
+ private static transient SparkSession instance = null;
+ public static SparkSession getInstance(SparkConf sparkConf) {
if (instance == null) {
- instance = new SQLContext(sparkContext);
+ instance = SparkSession.builder().config(sparkConf).getOrCreate();
}
return instance;
}