diff options
author | Reynold Xin <rxin@databricks.com> | 2016-11-04 23:34:29 -0700 |
---|---|---|
committer | Reynold Xin <rxin@databricks.com> | 2016-11-04 23:34:29 -0700 |
commit | 0f7c9e84e0d00813bf56712097677add5657f19f (patch) | |
tree | 29ca94f215b1d0c9a6201309f180c578a719edbd /sql/core/src | |
parent | 0e3312ee72c44f4c9acafbd80d0c8a14f3aff875 (diff) | |
download | spark-0f7c9e84e0d00813bf56712097677add5657f19f.tar.gz spark-0f7c9e84e0d00813bf56712097677add5657f19f.tar.bz2 spark-0f7c9e84e0d00813bf56712097677add5657f19f.zip |
[SPARK-18189] [SQL] [Followup] Move test from ReplSuite to prevent java.lang.ClassCircularityError
closes #15774
Diffstat (limited to 'sql/core/src')
-rw-r--r-- | sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala | 12 |
1 files changed, 12 insertions, 0 deletions
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala index 55f0487805..6fa7b04877 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala @@ -923,6 +923,18 @@ class DatasetSuite extends QueryTest with SharedSQLContext { .groupByKey(_.a).flatMapGroups { case (x, iter) => List[Int]() }) } + test("SPARK-18189: Fix serialization issue in KeyValueGroupedDataset") { + val resultValue = 12345 + val keyValueGrouped = Seq((1, 2), (3, 4)).toDS().groupByKey(_._1) + val mapGroups = keyValueGrouped.mapGroups((k, v) => (k, 1)) + val broadcasted = spark.sparkContext.broadcast(resultValue) + + // Using broadcast triggers serialization issue in KeyValueGroupedDataset + val dataset = mapGroups.map(_ => broadcasted.value) + + assert(dataset.collect() sameElements Array(resultValue, resultValue)) + } + Seq(true, false).foreach { eager => def testCheckpointing(testName: String)(f: => Unit): Unit = { test(s"Dataset.checkpoint() - $testName (eager = $eager)") { |