aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorWenchen Fan <wenchen@databricks.com>2016-01-17 09:11:43 -0800
committerDavies Liu <davies.liu@gmail.com>2016-01-17 09:11:43 -0800
commitcede7b2a1134a6c93aff20ed5625054d988d3659 (patch)
tree2364f0f142eaf0cf559ce21da6ead9a3cd2bb31e
parent3c0d2365d57fc49ac9bf0d7cc9bd2ef633fb5fb6 (diff)
downloadspark-cede7b2a1134a6c93aff20ed5625054d988d3659.tar.gz
spark-cede7b2a1134a6c93aff20ed5625054d988d3659.tar.bz2
spark-cede7b2a1134a6c93aff20ed5625054d988d3659.zip
[SPARK-12860] [SQL] speed up safe projection for primitive types
The idea is simple, use `SpecificMutableRow` instead of `GenericMutableRow` as result row for safe projection. A simple benchmark shows about 1.5x speed up for primitive types, code: https://gist.github.com/cloud-fan/fa77713ccebf0823b2ab#file-safeprojectionbenchmark-scala Author: Wenchen Fan <wenchen@databricks.com> Closes #10790 from cloud-fan/safe-projection.
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala5
1 files changed, 3 insertions, 2 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala
index e750ad9c18..4cb6af9d9f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala
@@ -165,7 +165,7 @@ object GenerateSafeProjection extends CodeGenerator[Seq[Expression], Projection]
public SpecificSafeProjection(Object[] references) {
this.references = references;
- mutableRow = new $genericMutableRowType(${expressions.size});
+ mutableRow = (MutableRow) references[references.length - 1];
${ctx.initMutableStates()}
}
@@ -180,6 +180,7 @@ object GenerateSafeProjection extends CodeGenerator[Seq[Expression], Projection]
logDebug(s"code for ${expressions.mkString(",")}:\n${CodeFormatter.format(code)}")
val c = CodeGenerator.compile(code)
- c.generate(ctx.references.toArray).asInstanceOf[Projection]
+ val resultRow = new SpecificMutableRow(expressions.map(_.dataType))
+ c.generate(ctx.references.toArray :+ resultRow).asInstanceOf[Projection]
}
}