diff options
author | OopsOutOfMemory <victorshengli@126.com> | 2015-02-06 13:20:10 -0800 |
---|---|---|
committer | Michael Armbrust <michael@databricks.com> | 2015-02-06 13:20:10 -0800 |
commit | 3d3ecd77411bfdd1c72be51616c46a6caf839be2 (patch) | |
tree | cd6fa2ad07f12e6e59c213ab0694c18090b54362 /repl/scala-2.10/src/main | |
parent | 4793c8402a19afe4df51129a7f99e07494a76af2 (diff) | |
download | spark-3d3ecd77411bfdd1c72be51616c46a6caf839be2.tar.gz spark-3d3ecd77411bfdd1c72be51616c46a6caf839be2.tar.bz2 spark-3d3ecd77411bfdd1c72be51616c46a6caf839be2.zip |
[SPARK-5586][Spark Shell][SQL] Make `sqlContext` available in spark shell
Result is like this
```
15/02/05 13:41:22 INFO SparkILoop: Created spark context..
Spark context available as sc.
15/02/05 13:41:22 INFO SparkILoop: Created sql context..
SQLContext available as sqlContext.
scala> sq
sql sqlContext sqlParser sqrt
```
Author: OopsOutOfMemory <victorshengli@126.com>
Closes #4387 from OopsOutOfMemory/sqlContextInShell and squashes the following commits:
c7f5203 [OopsOutOfMemory] auto-import sql() function
e160697 [OopsOutOfMemory] Merge branch 'sqlContextInShell' of https://github.com/OopsOutOfMemory/spark into sqlContextInShell
37c0a16 [OopsOutOfMemory] auto detect hive support
a9c59d9 [OopsOutOfMemory] rename and reduce range of imports
6b9e309 [OopsOutOfMemory] Merge branch 'master' into sqlContextInShell
cae652f [OopsOutOfMemory] make sqlContext available in spark shell
Diffstat (limited to 'repl/scala-2.10/src/main')
-rw-r--r-- | repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkILoop.scala | 19 | ||||
-rw-r--r-- | repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkILoopInit.scala | 10 |
2 files changed, 29 insertions, 0 deletions
diff --git a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkILoop.scala b/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkILoop.scala index 72c1a98999..b4db3df795 100644 --- a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkILoop.scala +++ b/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkILoop.scala @@ -45,6 +45,7 @@ import scala.reflect.api.{Mirror, TypeCreator, Universe => ApiUniverse} import org.apache.spark.Logging import org.apache.spark.SparkConf import org.apache.spark.SparkContext +import org.apache.spark.sql.SQLContext import org.apache.spark.util.Utils /** The Scala interactive shell. It provides a read-eval-print loop @@ -130,6 +131,7 @@ class SparkILoop( // NOTE: Must be public for visibility @DeveloperApi var sparkContext: SparkContext = _ + var sqlContext: SQLContext = _ override def echoCommandMessage(msg: String) { intp.reporter printMessage msg @@ -1016,6 +1018,23 @@ class SparkILoop( sparkContext } + @DeveloperApi + def createSQLContext(): SQLContext = { + val name = "org.apache.spark.sql.hive.HiveContext" + val loader = Utils.getContextOrSparkClassLoader + try { + sqlContext = loader.loadClass(name).getConstructor(classOf[SparkContext]) + .newInstance(sparkContext).asInstanceOf[SQLContext] + logInfo("Created sql context (with Hive support)..") + } + catch { + case cnf: java.lang.ClassNotFoundException => + sqlContext = new SQLContext(sparkContext) + logInfo("Created sql context..") + } + sqlContext + } + private def getMaster(): String = { val master = this.master match { case Some(m) => m diff --git a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkILoopInit.scala b/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkILoopInit.scala index 99bd777c04..0cf2de6d39 100644 --- a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkILoopInit.scala +++ b/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkILoopInit.scala @@ -127,7 +127,17 @@ private[repl] trait SparkILoopInit { _sc } """) + command(""" + @transient val sqlContext = { + val _sqlContext = org.apache.spark.repl.Main.interp.createSQLContext() + println("SQL context available as sqlContext.") + _sqlContext + } + """) command("import org.apache.spark.SparkContext._") + command("import sqlContext.implicits._") + command("import sqlContext.sql") + command("import org.apache.spark.sql.Dsl._") } } |