diff options
author | Koert Kuipers <koert@tresata.com> | 2016-05-03 18:06:35 -0700 |
---|---|---|
committer | Andrew Or <andrew@databricks.com> | 2016-05-03 18:06:35 -0700 |
commit | 9e4928b7e00788913553c1cb0722048001b91601 (patch) | |
tree | 3ec6e868a01046c775f0f4107be8c156f0558095 | |
parent | 0903a185c7ebc57c75301a27d215b08efd347f99 (diff) | |
download | spark-9e4928b7e00788913553c1cb0722048001b91601.tar.gz spark-9e4928b7e00788913553c1cb0722048001b91601.tar.bz2 spark-9e4928b7e00788913553c1cb0722048001b91601.zip |
[SPARK-15097][SQL] make Dataset.sqlContext a stable identifier for imports
## What changes were proposed in this pull request?
Make Dataset.sqlContext a lazy val so that its a stable identifier and can be used for imports.
Now this works again:
import someDataset.sqlContext.implicits._
## How was this patch tested?
Add unit test to DatasetSuite that uses the import show above.
Author: Koert Kuipers <koert@tresata.com>
Closes #12877 from koertkuipers/feat-sqlcontext-stable-import.
-rw-r--r-- | sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala | 3 | ||||
-rw-r--r-- | sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala | 13 |
2 files changed, 15 insertions, 1 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala index 08be94e8d4..1bea72c471 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala @@ -211,7 +211,8 @@ class Dataset[T] private[sql]( private implicit def classTag = unresolvedTEncoder.clsTag - def sqlContext: SQLContext = sparkSession.wrapped + // sqlContext must be val because a stable identifier is expected when you import implicits + @transient lazy val sqlContext: SQLContext = sparkSession.wrapped protected[sql] def resolve(colName: String): NamedExpression = { queryExecution.analyzed.resolveQuoted(colName, sparkSession.sessionState.analyzer.resolver) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala index eee21acf75..68a12b0622 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala @@ -653,6 +653,11 @@ class DatasetSuite extends QueryTest with SharedSQLContext { dataset.join(actual, dataset("user") === actual("id")).collect() } + + test("SPARK-15097: implicits on dataset's sqlContext can be imported") { + val dataset = Seq(1, 2, 3).toDS() + checkDataset(DatasetTransform.addOne(dataset), 2, 3, 4) + } } case class OtherTuple(_1: String, _2: Int) @@ -713,3 +718,11 @@ class JavaData(val a: Int) extends Serializable { object JavaData { def apply(a: Int): JavaData = new JavaData(a) } + +/** Used to test importing dataset.sqlContext.implicits._ */ +object DatasetTransform { + def addOne(ds: Dataset[Int]): Dataset[Int] = { + import ds.sqlContext.implicits._ + ds.map(_ + 1) + } +} |