diff options
author | Mridul Muralidharan <mridul@gmail.com> | 2013-04-22 08:01:13 +0530 |
---|---|---|
committer | Mridul Muralidharan <mridul@gmail.com> | 2013-04-22 08:01:13 +0530 |
commit | 7acab3ab45df421601ee9a076a61de00561a0308 (patch) | |
tree | ee2e9264e72bfd10e878c09765b5d580e382aca7 /examples/src | |
parent | ac2e8e8720f10efd640a67ad85270719ab2d43e9 (diff) | |
download | spark-7acab3ab45df421601ee9a076a61de00561a0308.tar.gz spark-7acab3ab45df421601ee9a076a61de00561a0308.tar.bz2 spark-7acab3ab45df421601ee9a076a61de00561a0308.zip |
Fix review comments, add a new api to SparkHadoopUtil to create appropriate Configuration. Modify an example to show how to use SplitInfo
Diffstat (limited to 'examples/src')
-rw-r--r-- | examples/src/main/scala/spark/examples/SparkHdfsLR.scala | 10 |
1 files changed, 8 insertions, 2 deletions
diff --git a/examples/src/main/scala/spark/examples/SparkHdfsLR.scala b/examples/src/main/scala/spark/examples/SparkHdfsLR.scala index 0f42f405a0..3d080a0257 100644 --- a/examples/src/main/scala/spark/examples/SparkHdfsLR.scala +++ b/examples/src/main/scala/spark/examples/SparkHdfsLR.scala @@ -4,6 +4,8 @@ import java.util.Random import scala.math.exp import spark.util.Vector import spark._ +import spark.deploy.SparkHadoopUtil +import spark.scheduler.InputFormatInfo /** * Logistic regression based classification. @@ -32,9 +34,13 @@ object SparkHdfsLR { System.err.println("Usage: SparkHdfsLR <master> <file> <iters>") System.exit(1) } + val inputPath = args(1) + val conf = SparkHadoopUtil.newConfiguration() val sc = new SparkContext(args(0), "SparkHdfsLR", - System.getenv("SPARK_HOME"), Seq(System.getenv("SPARK_EXAMPLES_JAR"))) - val lines = sc.textFile(args(1)) + System.getenv("SPARK_HOME"), Seq(System.getenv("SPARK_EXAMPLES_JAR")), Map(), + InputFormatInfo.computePreferredLocations( + Seq(new InputFormatInfo(conf, classOf[org.apache.hadoop.mapred.TextInputFormat], inputPath)))) + val lines = sc.textFile(inputPath) val points = lines.map(parsePoint _).cache() val ITERATIONS = args(2).toInt |