diff options
| author | Michael Armbrust <michael@databricks.com> | 2014-10-09 17:54:02 -0700 |
|---|---|---|
| committer | Michael Armbrust <michael@databricks.com> | 2014-10-09 17:54:02 -0700 |
| commit | 2837bf8548db7e9d43f6eefedf5a73feb22daedb (patch) | |
| tree | 111e32cdd175009cf9ecc0c8bebc755ab9dec905 /sql/core/src | |
| parent | 4e9b551a0b807f5a2cc6679165c8be4e88a3d077 (diff) | |
| download | spark-2837bf8548db7e9d43f6eefedf5a73feb22daedb.tar.gz spark-2837bf8548db7e9d43f6eefedf5a73feb22daedb.tar.bz2 spark-2837bf8548db7e9d43f6eefedf5a73feb22daedb.zip | |
[SPARK-3798][SQL] Store the output of a generator in a val
This prevents it from changing during serialization, leading to corrupted results.
Author: Michael Armbrust <michael@databricks.com>
Closes #2656 from marmbrus/generateBug and squashes the following commits:
efa32eb [Michael Armbrust] Store the output of a generator in a val. This prevents it from changing during serialization.
Diffstat (limited to 'sql/core/src')
| -rw-r--r-- | sql/core/src/main/scala/org/apache/spark/sql/execution/Generate.scala | 5 |
1 files changed, 3 insertions, 2 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/Generate.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/Generate.scala index c386fd121c..38877c28de 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/Generate.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/Generate.scala @@ -39,7 +39,8 @@ case class Generate( child: SparkPlan) extends UnaryNode { - protected def generatorOutput: Seq[Attribute] = { + // This must be a val since the generator output expr ids are not preserved by serialization. + protected val generatorOutput: Seq[Attribute] = { if (join && outer) { generator.output.map(_.withNullability(true)) } else { @@ -62,7 +63,7 @@ case class Generate( newProjection(child.output ++ nullValues, child.output) val joinProjection = - newProjection(child.output ++ generator.output, child.output ++ generator.output) + newProjection(child.output ++ generatorOutput, child.output ++ generatorOutput) val joinedRow = new JoinedRow iter.flatMap {row => |