diff options
| author | Michael Armbrust <michael@databricks.com> | 2017-03-08 01:32:42 -0800 |
|---|---|---|
| committer | Wenchen Fan <wenchen@databricks.com> | 2017-03-08 01:32:42 -0800 |
| commit | 314e48a3584bad4b486b046bbf0159d64ba857bc (patch) | |
| tree | 1a898c3bd09e1d795d812bd683725d7588a3056e | |
| parent | 56e1bd337ccb03cb01702e4260e4be59d2aa0ead (diff) | |
| download | spark-314e48a3584bad4b486b046bbf0159d64ba857bc.tar.gz spark-314e48a3584bad4b486b046bbf0159d64ba857bc.tar.bz2 spark-314e48a3584bad4b486b046bbf0159d64ba857bc.zip | |
[SPARK-18055][SQL] Use correct mirror in ExpresionEncoder
Previously, we were using the mirror of passed in `TypeTag` when reflecting to build an encoder. This fails when the outer class is built in (i.e. `Seq`'s default mirror is based on root classloader) but inner classes (i.e. `A` in `Seq[A]`) are defined in the REPL or a library.
This patch changes us to always reflect based on a mirror created using the context classloader.
Author: Michael Armbrust <michael@databricks.com>
Closes #17201 from marmbrus/replSeqEncoder.
| -rw-r--r-- | repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala | 11 | ||||
| -rw-r--r-- | sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala | 4 |
2 files changed, 13 insertions, 2 deletions
diff --git a/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala b/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala index 55c91675ed..121a02a9be 100644 --- a/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala +++ b/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala @@ -473,4 +473,15 @@ class ReplSuite extends SparkFunSuite { assertDoesNotContain("AssertionError", output) assertDoesNotContain("Exception", output) } + + test("newProductSeqEncoder with REPL defined class") { + val output = runInterpreterInPasteMode("local-cluster[1,4,4096]", + """ + |case class Click(id: Int) + |spark.implicits.newProductSeqEncoder[Click] + """.stripMargin) + + assertDoesNotContain("error:", output) + assertDoesNotContain("Exception", output) + } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala index 0782143d46..93fc565a53 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala @@ -45,8 +45,8 @@ import org.apache.spark.util.Utils object ExpressionEncoder { def apply[T : TypeTag](): ExpressionEncoder[T] = { // We convert the not-serializable TypeTag into StructType and ClassTag. - val mirror = typeTag[T].mirror - val tpe = typeTag[T].tpe + val mirror = ScalaReflection.mirror + val tpe = typeTag[T].in(mirror).tpe if (ScalaReflection.optionOfProductType(tpe)) { throw new UnsupportedOperationException( |