aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorReynold Xin <rxin@databricks.com>2017-04-10 14:14:09 -0700
committerReynold Xin <rxin@databricks.com>2017-04-10 14:14:09 -0700
commit379b0b0bbdbba2278ce3bcf471bd75f6ffd9cf0d (patch)
tree5f2aa5fcb403a39dfea11c249f521c6760cb3c06
parenta35b9d97123697d23fa0f691c1054f9adab5956c (diff)
downloadspark-379b0b0bbdbba2278ce3bcf471bd75f6ffd9cf0d.tar.gz
spark-379b0b0bbdbba2278ce3bcf471bd75f6ffd9cf0d.tar.bz2
spark-379b0b0bbdbba2278ce3bcf471bd75f6ffd9cf0d.zip
[SPARK-20283][SQL] Add preOptimizationBatches
## What changes were proposed in this pull request? We currently have postHocOptimizationBatches, but not preOptimizationBatches. This patch adds preOptimizationBatches so the optimizer debugging extensions are symmetric. ## How was this patch tested? N/A Author: Reynold Xin <rxin@databricks.com> Closes #17595 from rxin/SPARK-20283.
-rw-r--r--sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala8
1 files changed, 7 insertions, 1 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala
index 2cdfb7a782..1de4f508b8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala
@@ -30,7 +30,7 @@ class SparkOptimizer(
experimentalMethods: ExperimentalMethods)
extends Optimizer(catalog, conf) {
- override def batches: Seq[Batch] = (super.batches :+
+ override def batches: Seq[Batch] = (preOptimizationBatches ++ super.batches :+
Batch("Optimize Metadata Only Query", Once, OptimizeMetadataOnlyQuery(catalog, conf)) :+
Batch("Extract Python UDF from Aggregate", Once, ExtractPythonUDFFromAggregate) :+
Batch("Prune File Source Table Partitions", Once, PruneFileSourcePartitions)) ++
@@ -38,6 +38,12 @@ class SparkOptimizer(
Batch("User Provided Optimizers", fixedPoint, experimentalMethods.extraOptimizations: _*)
/**
+ * Optimization batches that are executed before the regular optimization batches (also before
+ * the finish analysis batch).
+ */
+ def preOptimizationBatches: Seq[Batch] = Nil
+
+ /**
* Optimization batches that are executed after the regular optimization batches, but before the
* batch executing the [[ExperimentalMethods]] optimizer rules. This hook can be used to add
* custom optimizer batches to the Spark optimizer.