diff options
| author | Dongjoon Hyun <dongjoon@apache.org> | 2016-04-19 21:38:15 +0800 |
|---|---|---|
| committer | Wenchen Fan <wenchen@databricks.com> | 2016-04-19 21:38:15 +0800 |
| commit | 3d46d796a3a2b60b37dc318652eded5e992be1e5 (patch) | |
| tree | a8157c731edb6f061fa163ef995bd4a839fccfdc /sql/core/src/main/scala/org/apache | |
| parent | 74fe235ab5ed169fb30d9d2c04077b90d1bf1b95 (diff) | |
| download | spark-3d46d796a3a2b60b37dc318652eded5e992be1e5.tar.gz spark-3d46d796a3a2b60b37dc318652eded5e992be1e5.tar.bz2 spark-3d46d796a3a2b60b37dc318652eded5e992be1e5.zip | |
[SPARK-14577][SQL] Add spark.sql.codegen.maxCaseBranches config option
## What changes were proposed in this pull request?
We currently disable codegen for `CaseWhen` if the number of branches is greater than 20 (in CaseWhen.MAX_NUM_CASES_FOR_CODEGEN). It would be better if this value is a non-public config defined in SQLConf.
## How was this patch tested?
Pass the Jenkins tests (including a new testcase `Support spark.sql.codegen.maxCaseBranches option`)
Author: Dongjoon Hyun <dongjoon@apache.org>
Closes #12353 from dongjoon-hyun/SPARK-14577.
Diffstat (limited to 'sql/core/src/main/scala/org/apache')
| -rw-r--r-- | sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegen.scala | 1 | ||||
| -rw-r--r-- | sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala | 8 |
2 files changed, 8 insertions, 1 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegen.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegen.scala index 29b66e3dee..46eaede5e7 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegen.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegen.scala @@ -429,7 +429,6 @@ case class CollapseCodegenStages(conf: SQLConf) extends Rule[SparkPlan] { private def supportCodegen(e: Expression): Boolean = e match { case e: LeafExpression => true - case e: CaseWhen => e.shouldCodegen // CodegenFallback requires the input to be an InternalRow case e: CodegenFallback => false case _ => true diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 7f206bdb9b..4ae8278a9d 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -402,6 +402,12 @@ object SQLConf { .intConf .createWithDefault(200) + val MAX_CASES_BRANCHES = SQLConfigBuilder("spark.sql.codegen.maxCaseBranches") + .internal() + .doc("The maximum number of switches supported with codegen.") + .intConf + .createWithDefault(20) + val FILES_MAX_PARTITION_BYTES = SQLConfigBuilder("spark.sql.files.maxPartitionBytes") .doc("The maximum number of bytes to pack into a single partition when reading files.") .longConf @@ -529,6 +535,8 @@ private[sql] class SQLConf extends Serializable with CatalystConf with Logging { def wholeStageMaxNumFields: Int = getConf(WHOLESTAGE_MAX_NUM_FIELDS) + def maxCaseBranchesForCodegen: Int = getConf(MAX_CASES_BRANCHES) + def exchangeReuseEnabled: Boolean = getConf(EXCHANGE_REUSE_ENABLED) def canonicalView: Boolean = getConf(CANONICAL_NATIVE_VIEW) |