aboutsummaryrefslogtreecommitdiff
path: root/examples/src/main/scala/org
diff options
context:
space:
mode:
authorCheng Lian <lian@databricks.com>2015-12-08 19:18:59 +0800
committerCheng Lian <lian@databricks.com>2015-12-08 19:18:59 +0800
commitda2012a0e152aa078bdd19a5c7f91786a2dd7016 (patch)
tree1f00975b821733925effbaf0090a40795c50d669 /examples/src/main/scala/org
parent037b7e76a7f8b59e031873a768d81417dd180472 (diff)
downloadspark-da2012a0e152aa078bdd19a5c7f91786a2dd7016.tar.gz
spark-da2012a0e152aa078bdd19a5c7f91786a2dd7016.tar.bz2
spark-da2012a0e152aa078bdd19a5c7f91786a2dd7016.zip
[SPARK-11551][DOC][EXAMPLE] Revert PR #10002
This reverts PR #10002, commit 78209b0ccaf3f22b5e2345dfb2b98edfdb746819. The original PR wasn't tested on Jenkins before being merged. Author: Cheng Lian <lian@databricks.com> Closes #10200 from liancheng/revert-pr-10002.
Diffstat (limited to 'examples/src/main/scala/org')
-rw-r--r--examples/src/main/scala/org/apache/spark/examples/ml/BinarizerExample.scala48
-rw-r--r--examples/src/main/scala/org/apache/spark/examples/ml/BucketizerExample.scala51
-rw-r--r--examples/src/main/scala/org/apache/spark/examples/ml/DCTExample.scala54
-rw-r--r--examples/src/main/scala/org/apache/spark/examples/ml/ElementWiseProductExample.scala53
-rw-r--r--examples/src/main/scala/org/apache/spark/examples/ml/MinMaxScalerExample.scala49
-rw-r--r--examples/src/main/scala/org/apache/spark/examples/ml/NGramExample.scala47
-rw-r--r--examples/src/main/scala/org/apache/spark/examples/ml/NormalizerExample.scala50
-rw-r--r--examples/src/main/scala/org/apache/spark/examples/ml/OneHotEncoderExample.scala58
-rw-r--r--examples/src/main/scala/org/apache/spark/examples/ml/PCAExample.scala54
-rw-r--r--examples/src/main/scala/org/apache/spark/examples/ml/PolynomialExpansionExample.scala53
-rw-r--r--examples/src/main/scala/org/apache/spark/examples/ml/RFormulaExample.scala49
-rw-r--r--examples/src/main/scala/org/apache/spark/examples/ml/StandardScalerExample.scala51
-rw-r--r--examples/src/main/scala/org/apache/spark/examples/ml/StopWordsRemoverExample.scala48
-rw-r--r--examples/src/main/scala/org/apache/spark/examples/ml/StringIndexerExample.scala49
-rw-r--r--examples/src/main/scala/org/apache/spark/examples/ml/TokenizerExample.scala54
-rw-r--r--examples/src/main/scala/org/apache/spark/examples/ml/VectorAssemblerExample.scala49
-rw-r--r--examples/src/main/scala/org/apache/spark/examples/ml/VectorIndexerExample.scala53
-rw-r--r--examples/src/main/scala/org/apache/spark/examples/ml/VectorSlicerExample.scala58
18 files changed, 0 insertions, 928 deletions
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/BinarizerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/BinarizerExample.scala
deleted file mode 100644
index e724aa5872..0000000000
--- a/examples/src/main/scala/org/apache/spark/examples/ml/BinarizerExample.scala
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package org.apache.spark.examples.ml
-
-// $example on$
-import org.apache.spark.ml.feature.Binarizer
-// $example off$
-import org.apache.spark.sql.{DataFrame, SQLContext}
-import org.apache.spark.{SparkConf, SparkContext}
-
-object BinarizerExample {
- def main(args: Array[String]): Unit = {
- val conf = new SparkConf().setAppName("BinarizerExample")
- val sc = new SparkContext(conf)
- val sqlContext = new SQLContext(sc)
- // $example on$
- val data = Array((0, 0.1), (1, 0.8), (2, 0.2))
- val dataFrame: DataFrame = sqlContext.createDataFrame(data).toDF("label", "feature")
-
- val binarizer: Binarizer = new Binarizer()
- .setInputCol("feature")
- .setOutputCol("binarized_feature")
- .setThreshold(0.5)
-
- val binarizedDataFrame = binarizer.transform(dataFrame)
- val binarizedFeatures = binarizedDataFrame.select("binarized_feature")
- binarizedFeatures.collect().foreach(println)
- // $example off$
- sc.stop()
- }
-}
-// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/BucketizerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/BucketizerExample.scala
deleted file mode 100644
index 30c2776d39..0000000000
--- a/examples/src/main/scala/org/apache/spark/examples/ml/BucketizerExample.scala
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package org.apache.spark.examples.ml
-
-// $example on$
-import org.apache.spark.ml.feature.Bucketizer
-// $example off$
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.{SparkConf, SparkContext}
-
-object BucketizerExample {
- def main(args: Array[String]): Unit = {
- val conf = new SparkConf().setAppName("BucketizerExample")
- val sc = new SparkContext(conf)
- val sqlContext = new SQLContext(sc)
-
- // $example on$
- val splits = Array(Double.NegativeInfinity, -0.5, 0.0, 0.5, Double.PositiveInfinity)
-
- val data = Array(-0.5, -0.3, 0.0, 0.2)
- val dataFrame = sqlContext.createDataFrame(data.map(Tuple1.apply)).toDF("features")
-
- val bucketizer = new Bucketizer()
- .setInputCol("features")
- .setOutputCol("bucketedFeatures")
- .setSplits(splits)
-
- // Transform original data into its bucket index.
- val bucketedData = bucketizer.transform(dataFrame)
- // $example off$
- sc.stop()
- }
-}
-// scalastyle:on println
-
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/DCTExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/DCTExample.scala
deleted file mode 100644
index 314c2c28a2..0000000000
--- a/examples/src/main/scala/org/apache/spark/examples/ml/DCTExample.scala
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package org.apache.spark.examples.ml
-
-// $example on$
-import org.apache.spark.ml.feature.DCT
-import org.apache.spark.mllib.linalg.Vectors
-// $example off$
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.{SparkConf, SparkContext}
-
-object DCTExample {
- def main(args: Array[String]): Unit = {
- val conf = new SparkConf().setAppName("DCTExample")
- val sc = new SparkContext(conf)
- val sqlContext = new SQLContext(sc)
-
- // $example on$
- val data = Seq(
- Vectors.dense(0.0, 1.0, -2.0, 3.0),
- Vectors.dense(-1.0, 2.0, 4.0, -7.0),
- Vectors.dense(14.0, -2.0, -5.0, 1.0))
-
- val df = sqlContext.createDataFrame(data.map(Tuple1.apply)).toDF("features")
-
- val dct = new DCT()
- .setInputCol("features")
- .setOutputCol("featuresDCT")
- .setInverse(false)
-
- val dctDf = dct.transform(df)
- dctDf.select("featuresDCT").show(3)
- // $example off$
- sc.stop()
- }
-}
-// scalastyle:on println
-
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/ElementWiseProductExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/ElementWiseProductExample.scala
deleted file mode 100644
index ac50bb7b2b..0000000000
--- a/examples/src/main/scala/org/apache/spark/examples/ml/ElementWiseProductExample.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package org.apache.spark.examples.ml
-
-// $example on$
-import org.apache.spark.ml.feature.ElementwiseProduct
-import org.apache.spark.mllib.linalg.Vectors
-// $example off$
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.{SparkConf, SparkContext}
-
-object ElementwiseProductExample {
- def main(args: Array[String]): Unit = {
- val conf = new SparkConf().setAppName("ElementwiseProductExample")
- val sc = new SparkContext(conf)
- val sqlContext = new SQLContext(sc)
-
- // $example on$
- // Create some vector data; also works for sparse vectors
- val dataFrame = sqlContext.createDataFrame(Seq(
- ("a", Vectors.dense(1.0, 2.0, 3.0)),
- ("b", Vectors.dense(4.0, 5.0, 6.0)))).toDF("id", "vector")
-
- val transformingVector = Vectors.dense(0.0, 1.0, 2.0)
- val transformer = new ElementwiseProduct()
- .setScalingVec(transformingVector)
- .setInputCol("vector")
- .setOutputCol("transformedVector")
-
- // Batch transform the vectors to create new column:
- transformer.transform(dataFrame).show()
- // $example off$
- sc.stop()
- }
-}
-// scalastyle:on println
-
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/MinMaxScalerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/MinMaxScalerExample.scala
deleted file mode 100644
index dac3679a5b..0000000000
--- a/examples/src/main/scala/org/apache/spark/examples/ml/MinMaxScalerExample.scala
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package org.apache.spark.examples.ml
-
-// $example on$
-import org.apache.spark.ml.feature.MinMaxScaler
-// $example off$
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.{SparkConf, SparkContext}
-
-object MinMaxScalerExample {
- def main(args: Array[String]): Unit = {
- val conf = new SparkConf().setAppName("MinMaxScalerExample")
- val sc = new SparkContext(conf)
- val sqlContext = new SQLContext(sc)
-
- // $example on$
- val dataFrame = sqlContext.read.format("libsvm").load("data/mllib/sample_libsvm_data.txt")
-
- val scaler = new MinMaxScaler()
- .setInputCol("features")
- .setOutputCol("scaledFeatures")
-
- // Compute summary statistics and generate MinMaxScalerModel
- val scalerModel = scaler.fit(dataFrame)
-
- // rescale each feature to range [min, max].
- val scaledData = scalerModel.transform(dataFrame)
- // $example off$
- sc.stop()
- }
-}
-// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/NGramExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/NGramExample.scala
deleted file mode 100644
index 8a85f71b56..0000000000
--- a/examples/src/main/scala/org/apache/spark/examples/ml/NGramExample.scala
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package org.apache.spark.examples.ml
-
-// $example on$
-import org.apache.spark.ml.feature.NGram
-// $example off$
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.{SparkConf, SparkContext}
-
-object NGramExample {
- def main(args: Array[String]): Unit = {
- val conf = new SparkConf().setAppName("NGramExample")
- val sc = new SparkContext(conf)
- val sqlContext = new SQLContext(sc)
-
- // $example on$
- val wordDataFrame = sqlContext.createDataFrame(Seq(
- (0, Array("Hi", "I", "heard", "about", "Spark")),
- (1, Array("I", "wish", "Java", "could", "use", "case", "classes")),
- (2, Array("Logistic", "regression", "models", "are", "neat"))
- )).toDF("label", "words")
-
- val ngram = new NGram().setInputCol("words").setOutputCol("ngrams")
- val ngramDataFrame = ngram.transform(wordDataFrame)
- ngramDataFrame.take(3).map(_.getAs[Stream[String]]("ngrams").toList).foreach(println)
- // $example off$
- sc.stop()
- }
-}
-// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/NormalizerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/NormalizerExample.scala
deleted file mode 100644
index 17571f0aad..0000000000
--- a/examples/src/main/scala/org/apache/spark/examples/ml/NormalizerExample.scala
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package org.apache.spark.examples.ml
-
-// $example on$
-import org.apache.spark.ml.feature.Normalizer
-// $example off$
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.{SparkConf, SparkContext}
-
-object NormalizerExample {
- def main(args: Array[String]): Unit = {
- val conf = new SparkConf().setAppName("NormalizerExample")
- val sc = new SparkContext(conf)
- val sqlContext = new SQLContext(sc)
-
- // $example on$
- val dataFrame = sqlContext.read.format("libsvm").load("data/mllib/sample_libsvm_data.txt")
-
- // Normalize each Vector using $L^1$ norm.
- val normalizer = new Normalizer()
- .setInputCol("features")
- .setOutputCol("normFeatures")
- .setP(1.0)
-
- val l1NormData = normalizer.transform(dataFrame)
-
- // Normalize each Vector using $L^\infty$ norm.
- val lInfNormData = normalizer.transform(dataFrame, normalizer.p -> Double.PositiveInfinity)
- // $example off$
- sc.stop()
- }
-}
-// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/OneHotEncoderExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/OneHotEncoderExample.scala
deleted file mode 100644
index 4512736943..0000000000
--- a/examples/src/main/scala/org/apache/spark/examples/ml/OneHotEncoderExample.scala
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package org.apache.spark.examples.ml
-
-// $example on$
-import org.apache.spark.ml.feature.{OneHotEncoder, StringIndexer}
-// $example off$
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.{SparkConf, SparkContext}
-
-object OneHotEncoderExample {
- def main(args: Array[String]): Unit = {
- val conf = new SparkConf().setAppName("OneHotEncoderExample")
- val sc = new SparkContext(conf)
- val sqlContext = new SQLContext(sc)
-
- // $example on$
- val df = sqlContext.createDataFrame(Seq(
- (0, "a"),
- (1, "b"),
- (2, "c"),
- (3, "a"),
- (4, "a"),
- (5, "c")
- )).toDF("id", "category")
-
- val indexer = new StringIndexer()
- .setInputCol("category")
- .setOutputCol("categoryIndex")
- .fit(df)
- val indexed = indexer.transform(df)
-
- val encoder = new OneHotEncoder().setInputCol("categoryIndex").
- setOutputCol("categoryVec")
- val encoded = encoder.transform(indexed)
- encoded.select("id", "categoryVec").foreach(println)
- // $example off$
- sc.stop()
- }
-}
-// scalastyle:on println
-
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/PCAExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/PCAExample.scala
deleted file mode 100644
index a18d4f3397..0000000000
--- a/examples/src/main/scala/org/apache/spark/examples/ml/PCAExample.scala
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package org.apache.spark.examples.ml
-
-// $example on$
-import org.apache.spark.ml.feature.PCA
-import org.apache.spark.mllib.linalg.Vectors
-// $example off$
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.{SparkConf, SparkContext}
-
-object PCAExample {
- def main(args: Array[String]): Unit = {
- val conf = new SparkConf().setAppName("PCAExample")
- val sc = new SparkContext(conf)
- val sqlContext = new SQLContext(sc)
-
- // $example on$
- val data = Array(
- Vectors.sparse(5, Seq((1, 1.0), (3, 7.0))),
- Vectors.dense(2.0, 0.0, 3.0, 4.0, 5.0),
- Vectors.dense(4.0, 0.0, 0.0, 6.0, 7.0)
- )
- val df = sqlContext.createDataFrame(data.map(Tuple1.apply)).toDF("features")
- val pca = new PCA()
- .setInputCol("features")
- .setOutputCol("pcaFeatures")
- .setK(3)
- .fit(df)
- val pcaDF = pca.transform(df)
- val result = pcaDF.select("pcaFeatures")
- result.show()
- // $example off$
- sc.stop()
- }
-}
-// scalastyle:on println
-
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/PolynomialExpansionExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/PolynomialExpansionExample.scala
deleted file mode 100644
index b8e9e6952a..0000000000
--- a/examples/src/main/scala/org/apache/spark/examples/ml/PolynomialExpansionExample.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package org.apache.spark.examples.ml
-
-// $example on$
-import org.apache.spark.ml.feature.PolynomialExpansion
-import org.apache.spark.mllib.linalg.Vectors
-// $example off$
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.{SparkConf, SparkContext}
-
-object PolynomialExpansionExample {
- def main(args: Array[String]): Unit = {
- val conf = new SparkConf().setAppName("PolynomialExpansionExample")
- val sc = new SparkContext(conf)
- val sqlContext = new SQLContext(sc)
-
- // $example on$
- val data = Array(
- Vectors.dense(-2.0, 2.3),
- Vectors.dense(0.0, 0.0),
- Vectors.dense(0.6, -1.1)
- )
- val df = sqlContext.createDataFrame(data.map(Tuple1.apply)).toDF("features")
- val polynomialExpansion = new PolynomialExpansion()
- .setInputCol("features")
- .setOutputCol("polyFeatures")
- .setDegree(3)
- val polyDF = polynomialExpansion.transform(df)
- polyDF.select("polyFeatures").take(3).foreach(println)
- // $example off$
- sc.stop()
- }
-}
-// scalastyle:on println
-
-
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/RFormulaExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/RFormulaExample.scala
deleted file mode 100644
index 286866edea..0000000000
--- a/examples/src/main/scala/org/apache/spark/examples/ml/RFormulaExample.scala
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package org.apache.spark.examples.ml
-
-// $example on$
-import org.apache.spark.ml.feature.RFormula
-// $example off$
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.{SparkConf, SparkContext}
-
-object RFormulaExample {
- def main(args: Array[String]): Unit = {
- val conf = new SparkConf().setAppName("RFormulaExample")
- val sc = new SparkContext(conf)
- val sqlContext = new SQLContext(sc)
-
- // $example on$
- val dataset = sqlContext.createDataFrame(Seq(
- (7, "US", 18, 1.0),
- (8, "CA", 12, 0.0),
- (9, "NZ", 15, 0.0)
- )).toDF("id", "country", "hour", "clicked")
- val formula = new RFormula()
- .setFormula("clicked ~ country + hour")
- .setFeaturesCol("features")
- .setLabelCol("label")
- val output = formula.fit(dataset).transform(dataset)
- output.select("features", "label").show()
- // $example off$
- sc.stop()
- }
-}
-// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/StandardScalerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/StandardScalerExample.scala
deleted file mode 100644
index 646ce0f13e..0000000000
--- a/examples/src/main/scala/org/apache/spark/examples/ml/StandardScalerExample.scala
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package org.apache.spark.examples.ml
-
-// $example on$
-import org.apache.spark.ml.feature.StandardScaler
-// $example off$
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.{SparkConf, SparkContext}
-
-object StandardScalerExample {
- def main(args: Array[String]): Unit = {
- val conf = new SparkConf().setAppName("StandardScalerExample")
- val sc = new SparkContext(conf)
- val sqlContext = new SQLContext(sc)
-
- // $example on$
- val dataFrame = sqlContext.read.format("libsvm").load("data/mllib/sample_libsvm_data.txt")
-
- val scaler = new StandardScaler()
- .setInputCol("features")
- .setOutputCol("scaledFeatures")
- .setWithStd(true)
- .setWithMean(false)
-
- // Compute summary statistics by fitting the StandardScaler.
- val scalerModel = scaler.fit(dataFrame)
-
- // Normalize each feature to have unit standard deviation.
- val scaledData = scalerModel.transform(dataFrame)
- // $example off$
- sc.stop()
- }
-}
-// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/StopWordsRemoverExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/StopWordsRemoverExample.scala
deleted file mode 100644
index 655ffce08d..0000000000
--- a/examples/src/main/scala/org/apache/spark/examples/ml/StopWordsRemoverExample.scala
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package org.apache.spark.examples.ml
-
-// $example on$
-import org.apache.spark.ml.feature.StopWordsRemover
-// $example off$
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.{SparkConf, SparkContext}
-
-object StopWordsRemoverExample {
- def main(args: Array[String]): Unit = {
- val conf = new SparkConf().setAppName("StopWordsRemoverExample")
- val sc = new SparkContext(conf)
- val sqlContext = new SQLContext(sc)
-
- // $example on$
- val remover = new StopWordsRemover()
- .setInputCol("raw")
- .setOutputCol("filtered")
-
- val dataSet = sqlContext.createDataFrame(Seq(
- (0, Seq("I", "saw", "the", "red", "baloon")),
- (1, Seq("Mary", "had", "a", "little", "lamb"))
- )).toDF("id", "raw")
-
- remover.transform(dataSet).show()
- // $example off$
- sc.stop()
- }
-}
-// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/StringIndexerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/StringIndexerExample.scala
deleted file mode 100644
index 1be8a5f33f..0000000000
--- a/examples/src/main/scala/org/apache/spark/examples/ml/StringIndexerExample.scala
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package org.apache.spark.examples.ml
-
-// $example on$
-import org.apache.spark.ml.feature.StringIndexer
-// $example off$
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.{SparkConf, SparkContext}
-
-object StringIndexerExample {
- def main(args: Array[String]): Unit = {
- val conf = new SparkConf().setAppName("StringIndexerExample")
- val sc = new SparkContext(conf)
- val sqlContext = new SQLContext(sc)
-
- // $example on$
- val df = sqlContext.createDataFrame(
- Seq((0, "a"), (1, "b"), (2, "c"), (3, "a"), (4, "a"), (5, "c"))
- ).toDF("id", "category")
-
- val indexer = new StringIndexer()
- .setInputCol("category")
- .setOutputCol("categoryIndex")
-
- val indexed = indexer.fit(df).transform(df)
- indexed.show()
- // $example off$
- sc.stop()
- }
-}
-// scalastyle:on println
-
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/TokenizerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/TokenizerExample.scala
deleted file mode 100644
index 01e0d1388a..0000000000
--- a/examples/src/main/scala/org/apache/spark/examples/ml/TokenizerExample.scala
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package org.apache.spark.examples.ml
-
-// $example on$
-import org.apache.spark.ml.feature.{RegexTokenizer, Tokenizer}
-// $example off$
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.{SparkConf, SparkContext}
-
-object TokenizerExample {
- def main(args: Array[String]): Unit = {
- val conf = new SparkConf().setAppName("TokenizerExample")
- val sc = new SparkContext(conf)
- val sqlContext = new SQLContext(sc)
-
- // $example on$
- val sentenceDataFrame = sqlContext.createDataFrame(Seq(
- (0, "Hi I heard about Spark"),
- (1, "I wish Java could use case classes"),
- (2, "Logistic,regression,models,are,neat")
- )).toDF("label", "sentence")
-
- val tokenizer = new Tokenizer().setInputCol("sentence").setOutputCol("words")
- val regexTokenizer = new RegexTokenizer()
- .setInputCol("sentence")
- .setOutputCol("words")
- .setPattern("\\W") // alternatively .setPattern("\\w+").setGaps(false)
-
- val tokenized = tokenizer.transform(sentenceDataFrame)
- tokenized.select("words", "label").take(3).foreach(println)
- val regexTokenized = regexTokenizer.transform(sentenceDataFrame)
- regexTokenized.select("words", "label").take(3).foreach(println)
- // $example off$
- sc.stop()
- }
-}
-// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/VectorAssemblerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/VectorAssemblerExample.scala
deleted file mode 100644
index d527924419..0000000000
--- a/examples/src/main/scala/org/apache/spark/examples/ml/VectorAssemblerExample.scala
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package org.apache.spark.examples.ml
-
-// $example on$
-import org.apache.spark.ml.feature.VectorAssembler
-import org.apache.spark.mllib.linalg.Vectors
-// $example off$
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.{SparkConf, SparkContext}
-
-object VectorAssemblerExample {
- def main(args: Array[String]): Unit = {
- val conf = new SparkConf().setAppName("VectorAssemblerExample")
- val sc = new SparkContext(conf)
- val sqlContext = new SQLContext(sc)
-
- // $example on$
- val dataset = sqlContext.createDataFrame(
- Seq((0, 18, 1.0, Vectors.dense(0.0, 10.0, 0.5), 1.0))
- ).toDF("id", "hour", "mobile", "userFeatures", "clicked")
-
- val assembler = new VectorAssembler()
- .setInputCols(Array("hour", "mobile", "userFeatures"))
- .setOutputCol("features")
-
- val output = assembler.transform(dataset)
- println(output.select("features", "clicked").first())
- // $example off$
- sc.stop()
- }
-}
-// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/VectorIndexerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/VectorIndexerExample.scala
deleted file mode 100644
index 14279d610f..0000000000
--- a/examples/src/main/scala/org/apache/spark/examples/ml/VectorIndexerExample.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package org.apache.spark.examples.ml
-
-// $example on$
-import org.apache.spark.ml.feature.VectorIndexer
-// $example off$
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.{SparkConf, SparkContext}
-
-object VectorIndexerExample {
- def main(args: Array[String]): Unit = {
- val conf = new SparkConf().setAppName("VectorIndexerExample")
- val sc = new SparkContext(conf)
- val sqlContext = new SQLContext(sc)
-
- // $example on$
- val data = sqlContext.read.format("libsvm").load("data/mllib/sample_libsvm_data.txt")
-
- val indexer = new VectorIndexer()
- .setInputCol("features")
- .setOutputCol("indexed")
- .setMaxCategories(10)
-
- val indexerModel = indexer.fit(data)
-
- val categoricalFeatures: Set[Int] = indexerModel.categoryMaps.keys.toSet
- println(s"Chose ${categoricalFeatures.size} categorical features: " +
- categoricalFeatures.mkString(", "))
-
- // Create new column "indexed" with categorical values transformed to indices
- val indexedData = indexerModel.transform(data)
- // $example off$
- sc.stop()
- }
-}
-// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/VectorSlicerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/VectorSlicerExample.scala
deleted file mode 100644
index 04f19829ef..0000000000
--- a/examples/src/main/scala/org/apache/spark/examples/ml/VectorSlicerExample.scala
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// scalastyle:off println
-package org.apache.spark.examples.ml
-
-// $example on$
-import org.apache.spark.ml.attribute.{Attribute, AttributeGroup, NumericAttribute}
-import org.apache.spark.ml.feature.VectorSlicer
-import org.apache.spark.mllib.linalg.Vectors
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.types.StructType
-// $example off$
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.{SparkConf, SparkContext}
-
-object VectorSlicerExample {
- def main(args: Array[String]): Unit = {
- val conf = new SparkConf().setAppName("VectorSlicerExample")
- val sc = new SparkContext(conf)
- val sqlContext = new SQLContext(sc)
-
- // $example on$
- val data = Array(Row(Vectors.dense(-2.0, 2.3, 0.0)))
-
- val defaultAttr = NumericAttribute.defaultAttr
- val attrs = Array("f1", "f2", "f3").map(defaultAttr.withName)
- val attrGroup = new AttributeGroup("userFeatures", attrs.asInstanceOf[Array[Attribute]])
-
- val dataRDD = sc.parallelize(data)
- val dataset = sqlContext.createDataFrame(dataRDD, StructType(Array(attrGroup.toStructField())))
-
- val slicer = new VectorSlicer().setInputCol("userFeatures").setOutputCol("features")
-
- slicer.setIndices(Array(1)).setNames(Array("f3"))
- // or slicer.setIndices(Array(1, 2)), or slicer.setNames(Array("f2", "f3"))
-
- val output = slicer.transform(dataset)
- println(output.select("userFeatures", "features").first())
- // $example off$
- sc.stop()
- }
-}
-// scalastyle:on println