aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala13
-rw-r--r--sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala20
2 files changed, 29 insertions, 4 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index c60af28b2a..0bcfbf6f84 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -187,10 +187,15 @@ class SQLContext(@transient val sparkContext: SparkContext)
/** Caches the specified table in-memory. */
def cacheTable(tableName: String): Unit = {
val currentTable = catalog.lookupRelation(None, tableName)
- val useCompression =
- sparkContext.conf.getBoolean("spark.sql.inMemoryColumnarStorage.compressed", false)
- val asInMemoryRelation =
- InMemoryRelation(useCompression, executePlan(currentTable).executedPlan)
+ val asInMemoryRelation = EliminateAnalysisOperators(currentTable.logicalPlan) match {
+ case _: InMemoryRelation =>
+ currentTable.logicalPlan
+
+ case _ =>
+ val useCompression =
+ sparkContext.conf.getBoolean("spark.sql.inMemoryColumnarStorage.compressed", false)
+ InMemoryRelation(useCompression, executePlan(currentTable).executedPlan)
+ }
catalog.registerTable(None, tableName, asInMemoryRelation)
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index e9360b0fc7..cca58c0063 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -17,7 +17,9 @@
package org.apache.spark.sql
+import org.apache.spark.sql.catalyst.analysis.EliminateAnalysisOperators
import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.columnar.{InMemoryColumnarTableScan, InMemoryRelation}
import org.apache.spark.sql.test._
/* Implicits */
@@ -405,4 +407,22 @@ class SQLQuerySuite extends QueryTest {
clear()
}
+ test("SPARK-1669: cacheTable should be idempotent") {
+ assume(!table("testData").logicalPlan.isInstanceOf[InMemoryRelation])
+
+ cacheTable("testData")
+ EliminateAnalysisOperators(table("testData").logicalPlan) match {
+ case _: InMemoryRelation =>
+ case _ =>
+ fail("testData should be cached")
+ }
+
+ cacheTable("testData")
+ EliminateAnalysisOperators(table("testData").logicalPlan) match {
+ case InMemoryRelation(_, _, _: InMemoryColumnarTableScan) =>
+ fail("cacheTable is not idempotent")
+
+ case _ =>
+ }
+ }
}