diff options
author | Marcelo Vanzin <vanzin@cloudera.com> | 2015-04-28 13:49:29 -0700 |
---|---|---|
committer | Andrew Or <andrew@databricks.com> | 2015-04-28 13:49:29 -0700 |
commit | 28b1af7420e0b5e7e2dfc09eafc45fe2ffcde5ec (patch) | |
tree | 9cc10fd691e87b26039e87f8ced7c37c0be131f7 | |
parent | 53befacced828bbac53c6e3a4976ec3f036bae9e (diff) | |
download | spark-28b1af7420e0b5e7e2dfc09eafc45fe2ffcde5ec.tar.gz spark-28b1af7420e0b5e7e2dfc09eafc45fe2ffcde5ec.tar.bz2 spark-28b1af7420e0b5e7e2dfc09eafc45fe2ffcde5ec.zip |
[MINOR] [CORE] Warn users who try to cache RDDs with dynamic allocation on.
Author: Marcelo Vanzin <vanzin@cloudera.com>
Closes #5751 from vanzin/cached-rdd-warning and squashes the following commits:
554cc07 [Marcelo Vanzin] Change message.
9efb9da [Marcelo Vanzin] [minor] [core] Warn users who try to cache RDDs with dynamic allocation on.
-rw-r--r-- | core/src/main/scala/org/apache/spark/SparkContext.scala | 5 |
1 files changed, 5 insertions, 0 deletions
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 65b903a55d..d0cf2a8dd0 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -1396,6 +1396,11 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli * Register an RDD to be persisted in memory and/or disk storage */ private[spark] def persistRDD(rdd: RDD[_]) { + _executorAllocationManager.foreach { _ => + logWarning( + s"Dynamic allocation currently does not support cached RDDs. Cached data for RDD " + + s"${rdd.id} will be lost when executors are removed.") + } persistentRdds(rdd.id) = rdd } |