diff options
author | Davies Liu <davies@databricks.com> | 2016-05-10 22:29:41 -0700 |
---|---|---|
committer | Reynold Xin <rxin@databricks.com> | 2016-05-10 22:29:41 -0700 |
commit | 1fbe2785dff53a9eae5f13809091de7520a1e1b2 (patch) | |
tree | 460f860a7c4ba44ba43f441c76acac6ede590865 | |
parent | 66554596064303757b921ebef683c3506d749775 (diff) | |
download | spark-1fbe2785dff53a9eae5f13809091de7520a1e1b2.tar.gz spark-1fbe2785dff53a9eae5f13809091de7520a1e1b2.tar.bz2 spark-1fbe2785dff53a9eae5f13809091de7520a1e1b2.zip |
[SPARK-15255][SQL] limit the length of name for cached DataFrame
## What changes were proposed in this pull request?
We use the tree string of an SparkPlan as the name of cached DataFrame, that could be very long, cause the browser to be not responsive. This PR will limit the length of the name to 1000 characters.
## How was this patch tested?
Here is how the UI looks right now:
![ui](https://cloud.githubusercontent.com/assets/40902/15163355/d5640f9c-16bc-11e6-8655-809af8a4fed1.png)
Author: Davies Liu <davies@databricks.com>
Closes #13033 from davies/cache_name.
-rw-r--r-- | sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala | 6 |
1 files changed, 5 insertions, 1 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala index a36071a97b..009fbaa006 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala @@ -19,6 +19,8 @@ package org.apache.spark.sql.execution.columnar import scala.collection.mutable.ArrayBuffer +import org.apache.commons.lang.StringUtils + import org.apache.spark.{Accumulable, Accumulator} import org.apache.spark.network.util.JavaUtils import org.apache.spark.rdd.RDD @@ -177,7 +179,9 @@ private[sql] case class InMemoryRelation( } }.persist(storageLevel) - cached.setName(tableName.map(n => s"In-memory table $n").getOrElse(child.toString)) + cached.setName( + tableName.map(n => s"In-memory table $n") + .getOrElse(StringUtils.abbreviate(child.toString, 1024))) _cachedColumnBuffers = cached } |