diff options
author | Jakob Odersky <jodersky@gmail.com> | 2015-12-10 08:35:52 -0800 |
---|---|---|
committer | Marcelo Vanzin <vanzin@cloudera.com> | 2015-12-10 08:35:52 -0800 |
commit | db5165246f2888537dd0f3d4c5a515875c7358ed (patch) | |
tree | 465481b020c29b5680a5532026ed6b4818a5f016 /repl/scala-2.11 | |
parent | 76540b6df5370b463277d3498097b2cc2d2e97a8 (diff) | |
download | spark-db5165246f2888537dd0f3d4c5a515875c7358ed.tar.gz spark-db5165246f2888537dd0f3d4c5a515875c7358ed.tar.bz2 spark-db5165246f2888537dd0f3d4c5a515875c7358ed.zip |
[SPARK-11832][CORE] Process arguments in spark-shell for Scala 2.11
Process arguments passed to the spark-shell. Fixes running the spark-shell from within a build environment.
Author: Jakob Odersky <jodersky@gmail.com>
Closes #9824 from jodersky/shell-2.11.
Diffstat (limited to 'repl/scala-2.11')
-rw-r--r-- | repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala | 37 | ||||
-rw-r--r-- | repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala | 3 |
2 files changed, 27 insertions, 13 deletions
diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala index 627148df80..455a6b9a93 100644 --- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala +++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala @@ -31,24 +31,39 @@ object Main extends Logging { val tmp = System.getProperty("java.io.tmpdir") val rootDir = conf.get("spark.repl.classdir", tmp) val outputDir = Utils.createTempDir(rootDir) - val s = new Settings() - s.processArguments(List("-Yrepl-class-based", - "-Yrepl-outdir", s"${outputDir.getAbsolutePath}", - "-classpath", getAddedJars.mkString(File.pathSeparator)), true) // the creation of SecurityManager has to be lazy so SPARK_YARN_MODE is set if needed lazy val classServer = new HttpServer(conf, outputDir, new SecurityManager(conf)) var sparkContext: SparkContext = _ var sqlContext: SQLContext = _ var interp = new SparkILoop // this is a public var because tests reset it. + private var hasErrors = false + + private def scalaOptionError(msg: String): Unit = { + hasErrors = true + Console.err.println(msg) + } + def main(args: Array[String]) { - if (getMaster == "yarn-client") System.setProperty("SPARK_YARN_MODE", "true") - // Start the classServer and store its URI in a spark system property - // (which will be passed to executors so that they can connect to it) - classServer.start() - interp.process(s) // Repl starts and goes in loop of R.E.P.L - classServer.stop() - Option(sparkContext).map(_.stop) + + val interpArguments = List( + "-Yrepl-class-based", + "-Yrepl-outdir", s"${outputDir.getAbsolutePath}", + "-classpath", getAddedJars.mkString(File.pathSeparator) + ) ++ args.toList + + val settings = new Settings(scalaOptionError) + settings.processArguments(interpArguments, true) + + if (!hasErrors) { + if (getMaster == "yarn-client") System.setProperty("SPARK_YARN_MODE", "true") + // Start the classServer and store its URI in a spark system property + // (which will be passed to executors so that they can connect to it) + classServer.start() + interp.process(settings) // Repl starts and goes in loop of R.E.P.L + classServer.stop() + Option(sparkContext).map(_.stop) + } } def getAddedJars: Array[String] = { diff --git a/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala b/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala index bf8997998e..63f3688c9e 100644 --- a/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala +++ b/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala @@ -54,8 +54,7 @@ class ReplSuite extends SparkFunSuite { new SparkILoop(in, new PrintWriter(out)) } org.apache.spark.repl.Main.interp = interp - Main.s.processArguments(List("-classpath", classpath), true) - Main.main(Array()) // call main + Main.main(Array("-classpath", classpath)) // call main org.apache.spark.repl.Main.interp = null if (oldExecutorClasspath != null) { |