diff options
author | Wenchen Fan <wenchen@databricks.com> | 2016-01-17 09:11:43 -0800 |
---|---|---|
committer | Davies Liu <davies.liu@gmail.com> | 2016-01-17 09:11:43 -0800 |
commit | cede7b2a1134a6c93aff20ed5625054d988d3659 (patch) | |
tree | 2364f0f142eaf0cf559ce21da6ead9a3cd2bb31e /sql | |
parent | 3c0d2365d57fc49ac9bf0d7cc9bd2ef633fb5fb6 (diff) | |
download | spark-cede7b2a1134a6c93aff20ed5625054d988d3659.tar.gz spark-cede7b2a1134a6c93aff20ed5625054d988d3659.tar.bz2 spark-cede7b2a1134a6c93aff20ed5625054d988d3659.zip |
[SPARK-12860] [SQL] speed up safe projection for primitive types
The idea is simple, use `SpecificMutableRow` instead of `GenericMutableRow` as result row for safe projection.
A simple benchmark shows about 1.5x speed up for primitive types, code: https://gist.github.com/cloud-fan/fa77713ccebf0823b2ab#file-safeprojectionbenchmark-scala
Author: Wenchen Fan <wenchen@databricks.com>
Closes #10790 from cloud-fan/safe-projection.
Diffstat (limited to 'sql')
-rw-r--r-- | sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala | 5 |
1 files changed, 3 insertions, 2 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala index e750ad9c18..4cb6af9d9f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala @@ -165,7 +165,7 @@ object GenerateSafeProjection extends CodeGenerator[Seq[Expression], Projection] public SpecificSafeProjection(Object[] references) { this.references = references; - mutableRow = new $genericMutableRowType(${expressions.size}); + mutableRow = (MutableRow) references[references.length - 1]; ${ctx.initMutableStates()} } @@ -180,6 +180,7 @@ object GenerateSafeProjection extends CodeGenerator[Seq[Expression], Projection] logDebug(s"code for ${expressions.mkString(",")}:\n${CodeFormatter.format(code)}") val c = CodeGenerator.compile(code) - c.generate(ctx.references.toArray).asInstanceOf[Projection] + val resultRow = new SpecificMutableRow(expressions.map(_.dataType)) + c.generate(ctx.references.toArray :+ resultRow).asInstanceOf[Projection] } } |