diff options
author | azagrebin <azagrebin@gmail.com> | 2015-02-16 18:06:19 -0800 |
---|---|---|
committer | Andrew Or <andrew@databricks.com> | 2015-02-16 18:07:08 -0800 |
commit | d8c70fb6d2a44d62f3dae33fc77422c37e5ee6b3 (patch) | |
tree | 4520ad41e052a898f14e11ff85d954473c6a67e2 /repl | |
parent | c2a9a61764fcbef07a8d42e5f68a46692bd12d75 (diff) | |
download | spark-d8c70fb6d2a44d62f3dae33fc77422c37e5ee6b3.tar.gz spark-d8c70fb6d2a44d62f3dae33fc77422c37e5ee6b3.tar.bz2 spark-d8c70fb6d2a44d62f3dae33fc77422c37e5ee6b3.zip |
[SPARK-3340] Deprecate ADD_JARS and ADD_FILES
I created a patch that disables the environment variables.
Thereby scala or python shell log a warning message to notify user about the deprecation
with the following message:
scala: "ADD_JARS environment variable is deprecated, use --jar spark submit argument instead"
python: "Warning: ADD_FILES environment variable is deprecated, use --py-files argument instead"
Is it what is expected or the code associated with the variables should be just completely removed?
Should it be somewhere documented?
Author: azagrebin <azagrebin@gmail.com>
Closes #4616 from azagrebin/master and squashes the following commits:
bab1aa9 [azagrebin] [SPARK-3340] Deprecate ADD_JARS and ADD_FILES: minor readability issue
0643895 [azagrebin] [SPARK-3340] Deprecate ADD_JARS and ADD_FILES: add warning messages
42f0107 [azagrebin] [SPARK-3340] Deprecate ADD_JARS and ADD_FILES
(cherry picked from commit 16687651f05bde8ff2e2fcef100383168958bf7f)
Signed-off-by: Andrew Or <andrew@databricks.com>
Diffstat (limited to 'repl')
-rw-r--r-- | repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkILoop.scala | 7 | ||||
-rw-r--r-- | repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala | 3 |
2 files changed, 7 insertions, 3 deletions
diff --git a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkILoop.scala b/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkILoop.scala index b4db3df795..8dc0e0c965 100644 --- a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkILoop.scala +++ b/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkILoop.scala @@ -1064,15 +1064,16 @@ class SparkILoop( private def main(settings: Settings): Unit = process(settings) } -object SparkILoop { +object SparkILoop extends Logging { implicit def loopToInterpreter(repl: SparkILoop): SparkIMain = repl.intp private def echo(msg: String) = Console println msg def getAddedJars: Array[String] = { val envJars = sys.env.get("ADD_JARS") - val propJars = sys.props.get("spark.jars").flatMap { p => - if (p == "") None else Some(p) + if (envJars.isDefined) { + logWarning("ADD_JARS environment variable is deprecated, use --jar spark submit argument instead") } + val propJars = sys.props.get("spark.jars").flatMap { p => if (p == "") None else Some(p) } val jars = propJars.orElse(envJars).getOrElse("") Utils.resolveURIs(jars).split(",").filter(_.nonEmpty) } diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala index dc25692749..2210fbaafe 100644 --- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala +++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala @@ -51,6 +51,9 @@ object Main extends Logging { def getAddedJars: Array[String] = { val envJars = sys.env.get("ADD_JARS") + if (envJars.isDefined) { + logWarning("ADD_JARS environment variable is deprecated, use --jar spark submit argument instead") + } val propJars = sys.props.get("spark.jars").flatMap { p => if (p == "") None else Some(p) } val jars = propJars.orElse(envJars).getOrElse("") Utils.resolveURIs(jars).split(",").filter(_.nonEmpty) |