From cede7b2a1134a6c93aff20ed5625054d988d3659 Mon Sep 17 00:00:00 2001 From: Wenchen Fan Date: Sun, 17 Jan 2016 09:11:43 -0800 Subject: [SPARK-12860] [SQL] speed up safe projection for primitive types The idea is simple, use `SpecificMutableRow` instead of `GenericMutableRow` as result row for safe projection. A simple benchmark shows about 1.5x speed up for primitive types, code: https://gist.github.com/cloud-fan/fa77713ccebf0823b2ab#file-safeprojectionbenchmark-scala Author: Wenchen Fan Closes #10790 from cloud-fan/safe-projection. --- .../sql/catalyst/expressions/codegen/GenerateSafeProjection.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'sql') diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala index e750ad9c18..4cb6af9d9f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala @@ -165,7 +165,7 @@ object GenerateSafeProjection extends CodeGenerator[Seq[Expression], Projection] public SpecificSafeProjection(Object[] references) { this.references = references; - mutableRow = new $genericMutableRowType(${expressions.size}); + mutableRow = (MutableRow) references[references.length - 1]; ${ctx.initMutableStates()} } @@ -180,6 +180,7 @@ object GenerateSafeProjection extends CodeGenerator[Seq[Expression], Projection] logDebug(s"code for ${expressions.mkString(",")}:\n${CodeFormatter.format(code)}") val c = CodeGenerator.compile(code) - c.generate(ctx.references.toArray).asInstanceOf[Projection] + val resultRow = new SpecificMutableRow(expressions.map(_.dataType)) + c.generate(ctx.references.toArray :+ resultRow).asInstanceOf[Projection] } } -- cgit v1.2.3