diff options
| author | wangzhenhua <wangzhenhua@huawei.com> | 2017-01-10 22:34:44 -0800 |
|---|---|---|
| committer | Reynold Xin <rxin@databricks.com> | 2017-01-10 22:34:44 -0800 |
| commit | a6155135690433988aa0cbf22f260f52a235e9f5 (patch) | |
| tree | 560378be0cef306565de851135ac932c10db951b /sql/hive/src/test/scala/org | |
| parent | 3b19c74e71fd6af18047747843e962b5401db4d9 (diff) | |
| download | spark-a6155135690433988aa0cbf22f260f52a235e9f5.tar.gz spark-a6155135690433988aa0cbf22f260f52a235e9f5.tar.bz2 spark-a6155135690433988aa0cbf22f260f52a235e9f5.zip | |
[SPARK-19149][SQL] Unify two sets of statistics in LogicalPlan
## What changes were proposed in this pull request?
Currently we have two sets of statistics in LogicalPlan: a simple stats and a stats estimated by cbo, but the computing logic and naming are quite confusing, we need to unify these two sets of stats.
## How was this patch tested?
Just modify existing tests.
Author: wangzhenhua <wangzhenhua@huawei.com>
Author: Zhenhua Wang <wzh_zju@163.com>
Closes #16529 from wzhfy/unifyStats.
Diffstat (limited to 'sql/hive/src/test/scala/org')
| -rw-r--r-- | sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala | 10 |
1 files changed, 5 insertions, 5 deletions
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala index b040f26d28..0053aa1642 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala @@ -69,7 +69,7 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto assert(properties.get("totalSize").toLong <= 0, "external table totalSize must be <= 0") assert(properties.get("rawDataSize").toLong <= 0, "external table rawDataSize must be <= 0") - val sizeInBytes = relation.statistics.sizeInBytes + val sizeInBytes = relation.stats(conf).sizeInBytes assert(sizeInBytes === BigInt(file1.length() + file2.length())) } } finally { @@ -80,7 +80,7 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto test("analyze MetastoreRelations") { def queryTotalSize(tableName: String): BigInt = - spark.sessionState.catalog.lookupRelation(TableIdentifier(tableName)).statistics.sizeInBytes + spark.sessionState.catalog.lookupRelation(TableIdentifier(tableName)).stats(conf).sizeInBytes // Non-partitioned table sql("CREATE TABLE analyzeTable (key STRING, value STRING)").collect() @@ -481,7 +481,7 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto test("estimates the size of a test MetastoreRelation") { val df = sql("""SELECT * FROM src""") val sizes = df.queryExecution.analyzed.collect { case mr: MetastoreRelation => - mr.statistics.sizeInBytes + mr.stats(conf).sizeInBytes } assert(sizes.size === 1, s"Size wrong for:\n ${df.queryExecution}") assert(sizes(0).equals(BigInt(5812)), @@ -501,7 +501,7 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto // Assert src has a size smaller than the threshold. val sizes = df.queryExecution.analyzed.collect { - case r if ct.runtimeClass.isAssignableFrom(r.getClass) => r.statistics.sizeInBytes + case r if ct.runtimeClass.isAssignableFrom(r.getClass) => r.stats(conf).sizeInBytes } assert(sizes.size === 2 && sizes(0) <= spark.sessionState.conf.autoBroadcastJoinThreshold && sizes(1) <= spark.sessionState.conf.autoBroadcastJoinThreshold, @@ -557,7 +557,7 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto val sizes = df.queryExecution.analyzed.collect { case r if implicitly[ClassTag[MetastoreRelation]].runtimeClass .isAssignableFrom(r.getClass) => - r.statistics.sizeInBytes + r.stats(conf).sizeInBytes } assert(sizes.size === 2 && sizes(1) <= spark.sessionState.conf.autoBroadcastJoinThreshold && sizes(0) <= spark.sessionState.conf.autoBroadcastJoinThreshold, |