diff options
author | Davies Liu <davies@databricks.com> | 2016-01-28 13:51:55 -0800 |
---|---|---|
committer | Davies Liu <davies.liu@gmail.com> | 2016-01-28 13:51:55 -0800 |
commit | cc18a7199240bf3b03410c1ba6704fe7ce6ae38e (patch) | |
tree | c28ed508523f45751172903ecd63130f6a3868c5 /sql/catalyst | |
parent | 676803963fcc08aa988aa6f14be3751314e006ca (diff) | |
download | spark-cc18a7199240bf3b03410c1ba6704fe7ce6ae38e.tar.gz spark-cc18a7199240bf3b03410c1ba6704fe7ce6ae38e.tar.bz2 spark-cc18a7199240bf3b03410c1ba6704fe7ce6ae38e.zip |
[SPARK-13031] [SQL] cleanup codegen and improve test coverage
1. enable whole stage codegen during tests even there is only one operator supports that.
2. split doProduce() into two APIs: upstream() and doProduce()
3. generate prefix for fresh names of each operator
4. pass UnsafeRow to parent directly (avoid getters and create UnsafeRow again)
5. fix bugs and tests.
Author: Davies Liu <davies@databricks.com>
Closes #10944 from davies/gen_refactor.
Diffstat (limited to 'sql/catalyst')
2 files changed, 12 insertions, 3 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala index 2747c315ad..e6704cf8bb 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala @@ -145,13 +145,22 @@ class CodegenContext { private val curId = new java.util.concurrent.atomic.AtomicInteger() /** + * A prefix used to generate fresh name. + */ + var freshNamePrefix = "" + + /** * Returns a term name that is unique within this instance of a `CodeGenerator`. * * (Since we aren't in a macro context we do not seem to have access to the built in `freshName` * function.) */ - def freshName(prefix: String): String = { - s"$prefix${curId.getAndIncrement}" + def freshName(name: String): String = { + if (freshNamePrefix == "") { + s"$name${curId.getAndIncrement}" + } else { + s"${freshNamePrefix}_$name${curId.getAndIncrement}" + } } /** diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala index d9fe76133c..ec31db19b9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala @@ -93,7 +93,7 @@ object GenerateMutableProjection extends CodeGenerator[Seq[Expression], () => Mu // Can't call setNullAt on DecimalType, because we need to keep the offset s""" if (this.isNull_$i) { - ${ctx.setColumn("mutableRow", e.dataType, i, null)}; + ${ctx.setColumn("mutableRow", e.dataType, i, "null")}; } else { ${ctx.setColumn("mutableRow", e.dataType, i, s"this.value_$i")}; } |