diff options
author | Takuya UESHIN <ueshin@happy-camper.st> | 2016-04-21 21:17:56 -0700 |
---|---|---|
committer | Davies Liu <davies.liu@gmail.com> | 2016-04-21 21:17:56 -0700 |
commit | f1fdb23821b89623b592bfb3ef73d61afbe93b0a (patch) | |
tree | c4abd96e897de43207ae88b49fc47f8892088167 /sql/catalyst/src/test | |
parent | df1953f0df8b43136157a18bea05fd6750906f68 (diff) | |
download | spark-f1fdb23821b89623b592bfb3ef73d61afbe93b0a.tar.gz spark-f1fdb23821b89623b592bfb3ef73d61afbe93b0a.tar.bz2 spark-f1fdb23821b89623b592bfb3ef73d61afbe93b0a.zip |
[SPARK-14793] [SQL] Code generation for large complex type exceeds JVM size limit.
## What changes were proposed in this pull request?
Code generation for complex type, `CreateArray`, `CreateMap`, `CreateStruct`, `CreateNamedStruct`, exceeds JVM size limit for large elements.
We should split generated code into multiple `apply` functions if the complex types have large elements, like `UnsafeProjection` or others for large expressions.
## How was this patch tested?
I added some tests to check if the generated codes for the expressions exceed or not.
Author: Takuya UESHIN <ueshin@happy-camper.st>
Closes #12559 from ueshin/issues/SPARK-14793.
Diffstat (limited to 'sql/catalyst/src/test')
-rw-r--r-- | sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala | 57 |
1 files changed, 57 insertions, 0 deletions
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala index b682e7d2b1..2082cea0f6 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala @@ -22,6 +22,7 @@ import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.expressions.codegen._ +import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, GenericArrayData} import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.UTF8String import org.apache.spark.util.ThreadUtils @@ -80,6 +81,62 @@ class CodeGenerationSuite extends SparkFunSuite with ExpressionEvalHelper { assert(actual(0) == cases) } + test("SPARK-14793: split wide array creation into blocks due to JVM code size limit") { + val length = 5000 + val expressions = Seq(CreateArray(List.fill(length)(EqualTo(Literal(1), Literal(1))))) + val plan = GenerateMutableProjection.generate(expressions) + val actual = plan(new GenericMutableRow(length)).toSeq(expressions.map(_.dataType)) + val expected = Seq(new GenericArrayData(Seq.fill(length)(true))) + + if (!checkResult(actual, expected)) { + fail(s"Incorrect Evaluation: expressions: $expressions, actual: $actual, expected: $expected") + } + } + + test("SPARK-14793: split wide map creation into blocks due to JVM code size limit") { + val length = 5000 + val expressions = Seq(CreateMap( + List.fill(length)(EqualTo(Literal(1), Literal(1))).zipWithIndex.flatMap { + case (expr, i) => Seq(Literal(i), expr) + })) + val plan = GenerateMutableProjection.generate(expressions) + val actual = plan(new GenericMutableRow(length)).toSeq(expressions.map(_.dataType)) + val expected = Seq(new ArrayBasedMapData( + new GenericArrayData(0 until length), + new GenericArrayData(Seq.fill(length)(true)))) + + if (!checkResult(actual, expected)) { + fail(s"Incorrect Evaluation: expressions: $expressions, actual: $actual, expected: $expected") + } + } + + test("SPARK-14793: split wide struct creation into blocks due to JVM code size limit") { + val length = 5000 + val expressions = Seq(CreateStruct(List.fill(length)(EqualTo(Literal(1), Literal(1))))) + val plan = GenerateMutableProjection.generate(expressions) + val actual = plan(new GenericMutableRow(length)).toSeq(expressions.map(_.dataType)) + val expected = Seq(InternalRow(Seq.fill(length)(true): _*)) + + if (!checkResult(actual, expected)) { + fail(s"Incorrect Evaluation: expressions: $expressions, actual: $actual, expected: $expected") + } + } + + test("SPARK-14793: split wide named struct creation into blocks due to JVM code size limit") { + val length = 5000 + val expressions = Seq(CreateNamedStruct( + List.fill(length)(EqualTo(Literal(1), Literal(1))).flatMap { + expr => Seq(Literal(expr.toString), expr) + })) + val plan = GenerateMutableProjection.generate(expressions) + val actual = plan(new GenericMutableRow(length)).toSeq(expressions.map(_.dataType)) + val expected = Seq(InternalRow(Seq.fill(length)(true): _*)) + + if (!checkResult(actual, expected)) { + fail(s"Incorrect Evaluation: expressions: $expressions, actual: $actual, expected: $expected") + } + } + test("test generated safe and unsafe projection") { val schema = new StructType(Array( StructField("a", StringType, true), |