From 0b6db8c186183704feafd26b454fff58a0e31861 Mon Sep 17 00:00:00 2001 From: Luca Rosellini Date: Fri, 3 Jan 2014 12:57:06 +0100 Subject: Added ‘-i’ command line option to spark REPL. We had to create a new implementation of both scala.tools.nsc.CompilerCommand and scala.tools.nsc.Settings, because using scala.tools.nsc.GenericRunnerSettings would bring in other options (-howtorun, -save and -execute) which don’t make sense in Spark. Any new Spark specific command line option could now be added to org.apache.spark.repl.SparkRunnerSettings class. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since the behavior of loading a script from the command line should be the same as loading it using the “:load” command inside the shell, the script should be loaded when the SparkContext is available, that’s why we had to move the call to ‘loadfiles(settings)’ _after_ the call to postInitialization(). This still doesn’t work if ‘isAsync = true’. --- .../org/apache/spark/repl/SparkCommandLine.scala | 22 ++++++++++++++++++++++ .../scala/org/apache/spark/repl/SparkILoop.scala | 7 ++++--- .../apache/spark/repl/SparkRunnerSettings.scala | 17 +++++++++++++++++ 3 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 repl/src/main/scala/org/apache/spark/repl/SparkCommandLine.scala create mode 100644 repl/src/main/scala/org/apache/spark/repl/SparkRunnerSettings.scala (limited to 'repl/src') diff --git a/repl/src/main/scala/org/apache/spark/repl/SparkCommandLine.scala b/repl/src/main/scala/org/apache/spark/repl/SparkCommandLine.scala new file mode 100644 index 0000000000..acb1e4ceda --- /dev/null +++ b/repl/src/main/scala/org/apache/spark/repl/SparkCommandLine.scala @@ -0,0 +1,22 @@ +package org.apache.spark.repl + +import scala.tools.nsc.{Settings, CompilerCommand} +import scala.Predef._ + +/** + * Command class enabling Spark-specific command line options (provided by + * org.apache.spark.repl.SparkRunnerSettings). + * + * @author Luca Rosellini + */ +class SparkCommandLine(args: List[String], override val settings: Settings) + extends CompilerCommand(args, settings) { + + def this(args: List[String], error: String => Unit) { + this(args, new SparkRunnerSettings(error)) + } + + def this(args: List[String]) { + this(args, str => Console.println("Error: " + str)) + } +} diff --git a/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala b/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala index f108c70f21..14c3feb55c 100644 --- a/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala +++ b/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala @@ -835,7 +835,7 @@ class SparkILoop(in0: Option[BufferedReader], protected val out: JPrintWriter, // runs :load `file` on any files passed via -i def loadFiles(settings: Settings) = settings match { - case settings: GenericRunnerSettings => + case settings: SparkRunnerSettings => for (filename <- settings.loadfiles.value) { val cmd = ":load " + filename command(cmd) @@ -902,7 +902,6 @@ class SparkILoop(in0: Option[BufferedReader], protected val out: JPrintWriter, addThunk(printWelcome()) addThunk(initializeSpark()) - loadFiles(settings) // it is broken on startup; go ahead and exit if (intp.reporter.hasErrors) return false @@ -922,6 +921,8 @@ class SparkILoop(in0: Option[BufferedReader], protected val out: JPrintWriter, } // printWelcome() + loadFiles(settings) + try loop() catch AbstractOrMissingHandler() finally closeInterpreter() @@ -955,7 +956,7 @@ class SparkILoop(in0: Option[BufferedReader], protected val out: JPrintWriter, /** process command-line arguments and do as they request */ def process(args: Array[String]): Boolean = { - val command = new CommandLine(args.toList, echo) + val command = new SparkCommandLine(args.toList, msg => echo(msg)) def neededHelp(): String = (if (command.settings.help.value) command.usageMsg + "\n" else "") + (if (command.settings.Xhelp.value) command.xusageMsg + "\n" else "") diff --git a/repl/src/main/scala/org/apache/spark/repl/SparkRunnerSettings.scala b/repl/src/main/scala/org/apache/spark/repl/SparkRunnerSettings.scala new file mode 100644 index 0000000000..235a688334 --- /dev/null +++ b/repl/src/main/scala/org/apache/spark/repl/SparkRunnerSettings.scala @@ -0,0 +1,17 @@ +package org.apache.spark.repl + +import scala.tools.nsc.Settings + +/** + * scala.tools.nsc.Settings implementation adding Spark-specific REPL + * command line options. + * + * @author Luca Rosellini + */ +class SparkRunnerSettings(error: String => Unit) extends Settings(error){ + + val loadfiles = MultiStringSetting( + "-i", + "file", + "load a file (assumes the code is given interactively)") +} -- cgit v1.2.3 From 4689ce29fd506d001d15c863ab4fe29bfac90326 Mon Sep 17 00:00:00 2001 From: Luca Rosellini Date: Tue, 7 Jan 2014 09:44:24 +0100 Subject: Added license header and removed @author tag --- .../org/apache/spark/repl/SparkCommandLine.scala | 19 +++++++++++++++++-- .../org/apache/spark/repl/SparkRunnerSettings.scala | 19 +++++++++++++++++-- 2 files changed, 34 insertions(+), 4 deletions(-) (limited to 'repl/src') diff --git a/repl/src/main/scala/org/apache/spark/repl/SparkCommandLine.scala b/repl/src/main/scala/org/apache/spark/repl/SparkCommandLine.scala index acb1e4ceda..05816941b5 100644 --- a/repl/src/main/scala/org/apache/spark/repl/SparkCommandLine.scala +++ b/repl/src/main/scala/org/apache/spark/repl/SparkCommandLine.scala @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.spark.repl import scala.tools.nsc.{Settings, CompilerCommand} @@ -6,8 +23,6 @@ import scala.Predef._ /** * Command class enabling Spark-specific command line options (provided by * org.apache.spark.repl.SparkRunnerSettings). - * - * @author Luca Rosellini */ class SparkCommandLine(args: List[String], override val settings: Settings) extends CompilerCommand(args, settings) { diff --git a/repl/src/main/scala/org/apache/spark/repl/SparkRunnerSettings.scala b/repl/src/main/scala/org/apache/spark/repl/SparkRunnerSettings.scala index 235a688334..7fd5fbb424 100644 --- a/repl/src/main/scala/org/apache/spark/repl/SparkRunnerSettings.scala +++ b/repl/src/main/scala/org/apache/spark/repl/SparkRunnerSettings.scala @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.spark.repl import scala.tools.nsc.Settings @@ -5,8 +22,6 @@ import scala.tools.nsc.Settings /** * scala.tools.nsc.Settings implementation adding Spark-specific REPL * command line options. - * - * @author Luca Rosellini */ class SparkRunnerSettings(error: String => Unit) extends Settings(error){ -- cgit v1.2.3