aboutsummaryrefslogtreecommitdiff
path: root/sql
diff options
context:
space:
mode:
authorDavies Liu <davies.liu@gmail.com>2016-05-18 16:02:52 -0700
committerDavies Liu <davies.liu@gmail.com>2016-05-18 16:02:52 -0700
commit84b23453ddb0a97e3d81306de0a5dcb64f88bdd0 (patch)
tree79668eb799d2630e7d8a2662082ee9d754be4fd6 /sql
parentfc29b896dae08b957ed15fa681b46162600a4050 (diff)
downloadspark-84b23453ddb0a97e3d81306de0a5dcb64f88bdd0.tar.gz
spark-84b23453ddb0a97e3d81306de0a5dcb64f88bdd0.tar.bz2
spark-84b23453ddb0a97e3d81306de0a5dcb64f88bdd0.zip
Revert "[SPARK-15392][SQL] fix default value of size estimation of logical plan"
This reverts commit fc29b896dae08b957ed15fa681b46162600a4050.
Diffstat (limited to 'sql')
-rw-r--r--sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala3
-rw-r--r--sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala9
2 files changed, 2 insertions, 10 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index a7f461381b..7933d12e28 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -605,7 +605,8 @@ private[sql] class SQLConf extends Serializable with CatalystConf with Logging {
def enableRadixSort: Boolean = getConf(RADIX_SORT_ENABLED)
- def defaultSizeInBytes: Long = getConf(DEFAULT_SIZE_IN_BYTES, Long.MaxValue)
+ def defaultSizeInBytes: Long =
+ getConf(DEFAULT_SIZE_IN_BYTES, autoBroadcastJoinThreshold + 1L)
def isParquetBinaryAsString: Boolean = getConf(PARQUET_BINARY_AS_STRING)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index df029e44c5..f573abf859 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -1476,13 +1476,4 @@ class DataFrameSuite extends QueryTest with SharedSQLContext {
getMessage()
assert(e1.startsWith("Path does not exist"))
}
-
- test("SPARK-15392: DataFrame created from RDD should not be broadcasted") {
- val rdd = sparkContext.range(1, 100).map(i => Row(i, i))
- val df = spark.createDataFrame(rdd, new StructType().add("a", LongType).add("b", LongType))
- assert(df.queryExecution.analyzed.statistics.sizeInBytes >
- spark.wrapped.conf.autoBroadcastJoinThreshold)
- assert(df.selectExpr("a").queryExecution.analyzed.statistics.sizeInBytes >
- spark.wrapped.conf.autoBroadcastJoinThreshold)
- }
}